def ce_loss_with_uncertainty(ctx, pred, y_l, log_var):
r = F.randn(0., 1., log_var.shape)
r = F.pow_scalar(F.exp(log_var), 0.5) * r
h = pred + r
with nn.context_scope(ctx):
loss_ce = F.mean(F.softmax_cross_entropy(h, y_l))
return loss_ce
def resnet_model(ctx, x, inmaps=64, act=F.relu, test=False):
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(x, inmaps, kernel=(3, 3), pad=(1, 1), with_bias=False)
h = PF.batch_normalization(h, decay_rate=0.9, batch_stat=not test)
h = act(h)
h = res_unit(h, "conv2", act, False) # -> 32x32
h = res_unit(h, "conv3", act, True) # -> 16x16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv4", act, False) # -> 16x16
h = res_unit(h, "conv5", act, True) # -> 8x8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv6", act, False) # -> 8x8
h = res_unit(h, "conv7", act, True) # -> 4x4
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
h = res_unit(h, "conv8", act, False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, 10)
return pred
def inplace_function_test_helper(inputs,
func,
func_args=[],
func_kwargs={},
ctx=None,
rng=None):
if rng is None:
rng = np.random.RandomState(313)
if ctx is None:
ctx = nn.Context()
with nn.context_scope(ctx):
a_s = [inp * 1.0 for inp in inputs]
y = func(*(a_s + list(func_args)), inplace=False, **func_kwargs)
l = F.sum(y)
a_s_i = [inp * 1.0 for inp in inputs]
y_i = func(*(a_s_i + list(func_args)), inplace=True, **func_kwargs)
l_i = F.sum(y_i)
data = [(rng.randn(*inp.shape), rng.randn(*inp.shape)) for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l.forward()
l.backward()
grads = [inp.g.copy() for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l_i.forward()
l_i.backward()
grads_i = [inp.g.copy() for inp in inputs]
for g, g_i in zip(grads, grads_i):
assert np.allclose(g, g_i)
def inplace_function_test_helper(inputs, func, func_args=[], func_kwargs={}, ctx=None, rng=None):
if rng is None:
rng = np.random.RandomState(313)
if ctx is None:
ctx = nn.Context()
with nn.context_scope(ctx):
a_s = [inp * 1.0 for inp in inputs]
y = func(*(a_s + list(func_args)), inplace=False, **func_kwargs)
l = F.sum(y)
a_s_i = [inp * 1.0 for inp in inputs]
y_i = func(*(a_s_i + list(func_args)), inplace=True, **func_kwargs)
l_i = F.sum(y_i)
data = [(rng.randn(*inp.shape), rng.randn(*inp.shape)) for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l.forward()
l.backward()
grads = [inp.g.copy() for inp in inputs]
for i in range(len(data)):
inputs[i].d = data[i][0]
inputs[i].g = data[i][1]
l_i.forward()
l_i.backward()
grads_i = [inp.g.copy() for inp in inputs]
for g, g_i in zip(grads, grads_i):
assert np.allclose(g, g_i)
def sigma_regularization(ctx, log_var, one):
with nn.context_scope(ctx):
h = F.exp(log_var)
h = F.pow_scalar(h, 0.5)
h = F.mean(h, axis=1)
r = F.mean(F.squared_error(h, one))
return r
def cnn_ae_model_000(ctx, x, act=F.relu, test=False):
with nn.parameter_scope("ae"):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 32, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 32, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 32, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv03", 32, k=4, s=2, p=1, act=act, test=test) # 32 -> 16
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 64, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 64, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 64, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv13", 64, k=4, s=2, p=1, act=act, test=test) # 16 -> 8
if not test:
h = F.dropout(h)
# Deconvblock0
h = deconv_unit(h, "deconv00", 64, k=4, s=2, p=1, act=act, test=test) # 8 -> 16
h = deconv_unit(h, "deconv01", 64, k=3, s=1, p=1, act=act, test=test)
h = deconv_unit(h, "deconv02", 64, k=3, s=1, p=1, act=act, test=test)
h = deconv_unit(h, "deconv03", 64, k=3, s=1, p=1, act=act, test=test)
# Deconvblock 1
h = deconv_unit(h, "deconv10", 32, k=4, s=2, p=1, act=act, test=test) # 16 -> 32
h = deconv_unit(h, "deconv11", 32, k=3, s=1, p=1, act=act, test=test)
h = deconv_unit(h, "deconv12", 32, k=3, s=1, p=1, act=act, test=test)
h = deconv_unit(h, "deconv13", 3, k=3, s=1, p=1, act=None, test=test)
return h
def cnn_model_003(ctx, x, act=F.relu, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
# Convblock 3
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
h = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
return h
def sigma_regularization(ctx, log_var, one):
with nn.context_scope(ctx):
h = F.exp(log_var)
h = F.pow_scalar(h, 0.5)
b = log_var.shape[0]
r = F.sum(F.squared_error(h, one)) / b
return r
def ce_loss_with_uncertainty(ctx, pred, y_l, log_var):
r = F.randn(0., 1., log_var.shape)
r = F.pow_scalar(F.exp(log_var), 0.5) * r
h = pred + r
with nn.context_scope(ctx):
loss_ce = F.mean(F.softmax_cross_entropy(h, y_l))
return loss_ce
def cifar10_resnet23_prediction(ctx, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope("conv1"):
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = bn_dropout(h, "bn_dropout1", test)
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = bn_dropout(h, "bn_dropout2", test)
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = bn_dropout(h, "bn_dropout3", test)
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def kl_divergence(ctx, pred, label, log_var):
with nn.context_scope(ctx):
s = F.pow_scalar(F.exp(log_var), 0.5)
elms = softmax_with_temperature(ctx, label, s) \
* F.log(F.softmax(pred, axis=1))
loss = -F.mean(F.sum(elms, axis=1))
return loss
def ref_fused_convolution(x, weight, bias, beta, gamma, rmean, rvar, z,
base_axis, pad, stride, dilation, group,
channel_last, decay_rate, eps, batch_stat,
nonlinearity, nonlinearity_args):
with nn.context_scope(cpu_context):
graph = RefFusedConvolutionGraph(**locals())
return graph.get_output()
def sigmas_regularization(ctx, log_var0, log_var1):
with nn.context_scope(ctx):
h0 = F.exp(log_var0)
h0 = F.pow_scalar(h0, 0.5)
h1 = F.exp(log_var1)
h1 = F.pow_scalar(h1, 0.5)
r = F.mean(F.squared_error(h0, h1))
return r
def test_rand_forward(seed, ctx, func_name, low, high, shape):
with nn.context_scope(ctx):
o = F.rand(low, high, shape, seed=seed)
assert o.shape == tuple(shape)
assert o.parent.name == func_name
o.forward()
assert np.all(o.d < high)
assert np.all(o.d >= low)
def sr_loss_with_uncertainty(ctx, pred0, pred1, log_var0, log_var1):
#TODO: squared error/absolute error
s0 = F.exp(log_var0)
s1 = F.exp(log_var1)
squared_error = F.squared_error(pred0, pred1)
with nn.context_scope(ctx):
loss_sr = F.mean(squared_error * (1 / s0 + 1 / s1) + (s0 / s1 + s1 / s0)) * 0.5
return loss_sr
def sigmas_regularization(ctx, log_var0, log_var1):
with nn.context_scope(ctx):
h0 = F.exp(log_var0)
h0 = F.pow_scalar(h0, 0.5)
h1 = F.exp(log_var1)
h1 = F.pow_scalar(h1, 0.5)
r = F.mean(F.squared_error(h0, h1))
return r
def test_randint_forward(seed, ctx, func_name, low, high, shape):
with nn.context_scope(ctx):
o = F.randint(low, high, shape, seed=seed)
assert o.shape == tuple(shape)
assert o.parent.name == func_name
o.forward()
assert np.all(o.d < high)
assert np.all(o.d >= low)
def sr_loss_with_uncertainty(ctx, pred0, pred1, log_var0, log_var1):
#TODO: squared error/absolute error
s0 = F.exp(log_var0)
s1 = F.exp(log_var1)
squared_error = F.squared_error(pred0, pred1)
with nn.context_scope(ctx):
loss_sr = F.mean(squared_error * (1 / s0 + 1 / s1) + (s0 / s1 + s1 / s0)) * 0.5
return loss_sr
def test_gru(seed, num_layers, dropout, bidirectional, training, seq_len,
batch_size, input_size, hidden_size, with_bias, ctx, func_name):
from nbla_test_utils import function_tester
if func_name == "GRU":
pytest.skip("Not implemented in CPU.")
with nn.context_scope(ctx):
rng = np.random.RandomState(seed)
num_directions = 1
if bidirectional:
num_directions = 2
inputs = [
rng.randn(seq_len, batch_size, input_size).astype(np.float32)
]
inputs += [
rng.randn(num_layers, num_directions, batch_size,
hidden_size).astype(np.float32)
]
inputs += [
rng.randn(num_directions, 3, hidden_size, input_size + hidden_size)
]
if num_layers > 1:
inputs += [
rng.randn(max(1, num_layers - 1), num_directions, 3,
hidden_size, num_directions * hidden_size +
hidden_size).astype(np.float32)
]
else:
inputs += [None]
if with_bias:
inputs += [
rng.randn(num_layers, num_directions, 4,
hidden_size).astype(np.float32)
]
else:
inputs += [None]
backward = [False for _ in inputs]
if training:
backward = [True for _ in inputs]
function_tester(rng,
F.gru,
execute_fixed_length_gru,
inputs,
func_kwargs=dict(num_layers=num_layers,
dropout=dropout,
bidirectional=bidirectional,
training=training),
atol_f=1e-6,
atol_b=1e-2,
dstep=1e-3,
backward=backward,
ctx=ctx,
func_name=func_name,
ref_grad=get_gru_grad,
disable_half_test=True)
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act,
test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch,
"conv23",
10,
k=1,
s=1,
p=0,
act=act,
test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def cifar10_resnet23_prediction(ctx, scope, image, test=False):
"""
Construct ResNet 23
"""
# Residual Unit
def res_unit(x, scope_name, dn=False, test=False):
C = x.shape[1]
with nn.parameter_scope(scope_name):
# Conv -> BN -> Relu
with nn.parameter_scope("conv1"):
h = PF.convolution(x, C / 2, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN -> Relu
with nn.parameter_scope("conv2"):
h = PF.convolution(h, C / 2, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
# Conv -> BN
with nn.parameter_scope("conv3"):
h = PF.convolution(h, C, kernel=(1, 1), pad=(0, 0),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
# Residual -> Relu
h = F.relu(h + x)
# Maxpooling
if dn:
h = F.max_pooling(h, kernel=(2, 2), stride=(2, 2))
return h
# Random generator for using the same init parameters in all devices
nmaps = 64
ncls = 10
# Conv -> BN -> Relu
with nn.context_scope(ctx):
with nn.parameter_scope(scope):
with nn.parameter_scope("conv1"):
h = PF.convolution(image, nmaps, kernel=(3, 3), pad=(1, 1),
with_bias=False)
h = PF.batch_normalization(h, batch_stat=not test)
h = F.relu(h)
h = res_unit(h, "conv2", False) # -> 32x32
h = res_unit(h, "conv3", True) # -> 16x16
h = res_unit(h, "conv4", False) # -> 16x16
h = res_unit(h, "conv5", True) # -> 8x8
h = res_unit(h, "conv6", False) # -> 8x8
h = res_unit(h, "conv7", True) # -> 4x4
h = res_unit(h, "conv8", False) # -> 4x4
h = F.average_pooling(h, kernel=(4, 4)) # -> 1x1
pred = PF.affine(h, ncls)
return pred
def ref_fused_convolution(ctx, x, weight, bias, beta, gamma, rmean, rvar, z,
base_axis, pad, stride, dilation, group, channel_last,
decay_rate, eps, batch_stat,
nonlinearity, nonlinearity_args, pad_mode, constant_value):
args = locals().copy()
del args['ctx']
with nn.context_scope(ctx):
graph = RefFusedConvolutionGraph(**args)
return graph.get_output()
def test_image_augmentation_forward(seed, shape, ctx, func_name):
rng = np.random.RandomState(seed)
inputs = [rng.randn(16, 3, 8, 8).astype(np.float32)]
i = nn.Variable(inputs[0].shape)
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i)
assert o.d.shape == inputs[0].shape
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i, shape=shape, pad=(2, 2),
min_scale=0.8, max_scale=1.2, angle=0.2,
aspect_ratio=1.1, distortion=0.1,
flip_lr=True, flip_ud=False,
brightness=0.1, brightness_each=True,
contrast=1.1, contrast_center=0.5, contrast_each=True,
noise=0.1, seed=0)
assert o.d.shape == (inputs[0].shape[0],) + shape
def er_loss(ctx, pred):
with nn.context_scope(ctx):
bs = pred.shape[0]
d = np.prod(pred.shape[1:])
denominator = bs * d
pred_normalized = F.softmax(pred)
pred_log_normalized = F.log(F.softmax(pred))
loss_er = -F.sum(pred_normalized * pred_log_normalized) / denominator
return loss_er
def test_random_crop_forward_backward(seed, inshape, shape, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_crop(i, shape, 0, seed)
if shape is not None:
max_correl = 0
possible_crop_range = [
input - output for output, input in zip(shape, inshape)
]
for crop_pos in itertools.product(*map(
tuple,
map(lambda x: range(*x), [(0, r + 1)
for r in possible_crop_range]))):
r = inputs[0][crop_pos[0]:crop_pos[0] + shape[0],
crop_pos[1]:crop_pos[1] + shape[1],
crop_pos[2]:crop_pos[2] + shape[2]]
assert (o.d.shape == r.shape)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
max_correl = correl_and_p[0]
else:
max_correl = pearsonr(o.d.flatten(), inputs[0].flatten())[0]
assert (max_correl == 1.0)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_diff = rng.randn(*o.shape).astype(i.d.dtype)
o.backward(o_diff)
assert np.all(i.g == 0)
def er_loss(ctx, pred):
with nn.context_scope(ctx):
bs = pred.shape[0]
d = np.prod(pred.shape[1:])
denominator = bs * d
pred_normalized = F.softmax(pred)
pred_log_normalized = F.log(F.softmax(pred))
loss_er = - F.sum(pred_normalized * pred_log_normalized) / denominator
return loss_er
def test_batch_normalization_forward_backward(seed, axis, decay_rate, eps,
output_stat, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = list(create_inputs(rng, axis))
axes = [axis]
batch_stat = True
function_tester(rng,
F.batch_normalization,
ref_batch_normalization,
inputs,
func_args=[axes, decay_rate, eps, batch_stat, output_stat],
backward=[True, True, True, False, False],
ctx=ctx,
func_name=func_name,
dstep=1e-2,
atol_b=1e-2)
# Check if running mean and var works.
vinputs = []
for i in inputs:
vinputs.append(nn.Variable(i.shape, True))
vinputs[-1].d = i
for i in range(5):
inputs[0] = rng.randn(*inputs[0].shape)
vinputs[0].d[...] = inputs[0]
ref_y = ref_batch_normalization(
*(inputs + [axes, decay_rate, eps, batch_stat, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.batch_normalization(
*(vinputs + [axes, decay_rate, eps, batch_stat, output_stat]))
assert np.allclose(vinputs[3].d, inputs[3])
assert np.allclose(vinputs[4].d, inputs[4], atol=1e-3)
# Check if global stat mode works
batch_stat = False
if output_stat:
return
ref_y = ref_batch_normalization(
*(inputs + [axes, decay_rate, eps, batch_stat, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.batch_normalization(
*(vinputs + [axes, decay_rate, eps, batch_stat, output_stat]))
assert np.allclose(ref_y, y.d, atol=1e-6)
def get_model(args,
num_classes,
test=False,
channel_last=False,
mixup=None,
channels=4,
spatial_size=224,
label_smoothing=0,
ctx_for_loss=None):
"""
Create computation graph and variables.
"""
from models import build_network
from utils.loss import softmax_cross_entropy_with_label_smoothing
if hasattr(spatial_size, '__len__'):
assert len(spatial_size) == 2, \
f'Spatial size must be a scalar or a tuple of two ints. Given {spatial_size}'
spatial_shape = tuple(spatial_size)
else:
spatial_shape = (spatial_size, spatial_size)
if channel_last:
image = nn.Variable(
(args.batch_size, spatial_shape[0], spatial_shape[1], channels))
else:
image = nn.Variable((args.batch_size, channels) + spatial_shape)
label = nn.Variable([args.batch_size, 1])
in_image = image
in_label = label
if mixup is not None:
image, label = mixup.mix_data(image, label)
pred, hidden = build_network(image,
num_classes,
args.arch,
test=test,
channel_last=channel_last)
pred.persistent = True
def define_loss(pred, in_label, label, label_smoothing):
loss = F.mean(
softmax_cross_entropy_with_label_smoothing(pred, label,
label_smoothing))
error = F.sum(F.top_n_error(pred, in_label, n=1))
return loss, error
# Use specified context if possible.
# We use it when we pass float32 context to avoid nan issue
if ctx_for_loss is not None:
with nn.context_scope(ctx_for_loss):
loss, error = define_loss(pred, in_label, label, label_smoothing)
else:
loss, error = define_loss(pred, in_label, label, label_smoothing)
Model = namedtuple('Model',
['image', 'label', 'pred', 'loss', 'error', 'hidden'])
return Model(in_image, in_label, pred, loss, error, hidden)
def test_dropout_forward_backward(p, seed, ctx, func_name):
from nbla_test_utils import cap_ignore_region
# Note: each backward execution requires a forward execution in NNabla.
with nn.context_scope(ctx):
# Create inputs
rng = np.random.RandomState(seed)
inputs = [
cap_ignore_region(
rng.randn(2, 3, 4).astype(np.float32) * 2, (-1e-3, 1e-3))
] # Ensure there is no zero.
x = nn.Variable(inputs[0].shape, need_grad=True)
x.d = inputs[0]
init_dx = rng.randn(*x.shape).astype(x.data.dtype)
init_dy = rng.randn(*x.shape).astype(x.data.dtype)
# Construct graph
y = F.dropout(x, p)
# Reference parameter
scale = 1. / (1. - p)
# Test forward
y.forward(clear_buffer=True)
mask = (y.d != 0)
ref_y = x.d * mask * scale
assert_allclose(y.d, ref_y)
assert y.parent.name == func_name
# Test backward
x.g[...] = init_dx
y.backward(init_dy, clear_buffer=True)
ref_dx = init_dy * mask * scale
assert_allclose(x.g, init_dx + ref_dx)
# Test accumulation
y.forward(clear_no_need_grad=True)
mask = (y.d != 0)
x.g[...] = 1
y.g = init_dy
y.parent.backward([x], [y], [False])
ref_dx = init_dy * mask * scale
assert_allclose(x.g, ref_dx)
# Test accum=False with NaN gradient
y.forward(clear_no_need_grad=True)
x.g = np.float32('nan')
y.parent.backward([x], [y], [False])
assert not np.any(np.isnan(x.g))
# Test need_grad
y.forward(clear_no_need_grad=True)
x.g[...] = 0
x.need_grad = False
y.backward(init_dy)
assert np.all(x.g == 0)
def ref_grad_fused_convolution(x, weight, bias, beta, gamma, rmean, rvar, z, dy,
base_axis, pad, stride, dilation, group, channel_last,
decay_rate, eps, batch_stat,
nonlinearity, nonlinearity_args, need_grad_flags):
args = locals().copy()
del args['dy']
del args['need_grad_flags']
with nn.context_scope(cpu_context):
graph = RefFusedConvolutionGraph(**args)
return graph.get_grads(dy, need_grad_flags=need_grad_flags)
def test_function_context(seed):
rng = np.random.RandomState(313)
xd = rng.randn(2, 3)
x = nn.Variable.from_numpy_array(xd)
ctx1 = nn.Context(backend=['cpu:float'],
array_class='CpuCachedArray', device_id='1')
with nn.context_scope(ctx1):
y = F.relu(x)
ctx0 = nn.Context(backend=['cpu:float'],
array_class='CpuCachedArray', device_id='0')
# TODO: use id or hash if we determine the spec
assert str(ctx0) != str(ctx1)
assert str(ctx1) == str(y.parent.context)
with nn.context_scope(y.parent.context):
z = F.relu(x)
assert str(y.parent.context) == str(z.parent.context)
def test_randint_forward(seed, ctx, func_name, low, high, shape):
with nn.context_scope(ctx):
o = F.randint(low, high, shape, seed=seed)
assert o.shape == tuple(shape)
assert o.parent.name == func_name
o.forward()
# NOTE: The following should be < high,
# but use <= high because std::uniform_random contains a bug.
assert np.all(o.d <= high)
assert np.all(o.d >= low)
def sr_loss_with_uncertainty(ctx, pred0, pred1, log_var0, log_var1):
var0 = F.exp(log_var0)
var1 = F.exp(log_var1)
s0 = F.pow_scalar(var0, 0.5)
s1 = F.pow_scalar(var0, 0.5)
squared_error = F.squared_error(pred0, pred1)
with nn.context_scope(ctx):
loss = F.log(s1/s0) + (var0/var1 + squared_error/var1) * 0.5
loss_sr = F.mean(loss)
return loss_sr
def test_image_augmentation_forward(seed, ctx, func_name):
rng = np.random.RandomState(seed)
inputs = [rng.randn(16, 3, 8, 8).astype(np.float32)]
i = nn.Variable(inputs[0].shape)
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i)
assert o.d.shape == inputs[0].shape
shape = (3, 5, 8)
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i, shape=shape, pad=(2, 2),
min_scale=0.8, max_scale=1.2, angle=0.2,
aspect_ratio=1.1, distortion=0.1,
flip_lr=True, flip_ud=False,
brightness=0.1, brightness_each=True,
contrast=1.1, contrast_center=0.5, contrast_each=True,
noise=0.1, seed=0)
assert o.d.shape == (inputs[0].shape[0],) + shape
def test_pack_padded_long_sequence_forward_backward(total_length, padding_value,
batch_first, shapes, seed, ctx, func_name):
if not func_name.endswith("Cuda"):
pytest.skip(
"PackPaddedSequence tests except for Cuda for very long sequence skips.")
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
sequences = [rng.randn(*shape).astype(np.float32) for shape in shapes]
padded_sequence = pad_sequence(sequences, batch_first)
lengths = np.array([seq.shape[0] for seq in sequences])
inputs = [padded_sequence, lengths]
func_args0 = [batch_first]
func_args1 = [batch_first, padding_value, total_length]
insert_identity = [True, False]
# Forward
function_tester(rng, F.pack_padded_sequence, ref_pack_padded_sequence, inputs,
ctx=ctx, func_name=func_name, func_args=func_args0,
backward=[False, False],
atol_f=1e-3, atol_b=1e-2, insert_identity=insert_identity)
# Backward
import nnabla as nn
padded_sequence0 = nn.Variable.from_numpy_array(
inputs[0]).apply(need_grad=True)
lengths = nn.Variable.from_numpy_array(inputs[1])
with nn.context_scope(ctx), nn.auto_forward():
# Pack backward
padded_sequence0.g = rng.randn(*padded_sequence0.shape)
packed_sequence0, batch_sizes = F.pack_padded_sequence(
padded_sequence0, lengths, *func_args0)
g = rng.randn(*packed_sequence0.shape)
packed_sequence0.g = g
packed_sequence0.parent.backward([padded_sequence0, lengths], [packed_sequence0, batch_sizes],
[False, False])
# Unpack
packed_sequence1 = nn.Variable.from_numpy_array(g)
padded_sequence1, lengths = F.pad_packed_sequence(
packed_sequence1, batch_sizes, *func_args1)
# Compare w/o accum
np.testing.assert_allclose(padded_sequence0.g.flatten(),
padded_sequence1.d.flatten(
)[:np.prod(padded_sequence0.shape)],
atol=1e-4,
err_msg="{} test (w/o accum) with long sequence failed.".format(func_name))
# Compare w/ accum
packed_sequence0.parent.backward([padded_sequence0, lengths], [packed_sequence0, batch_sizes],
[True, False])
np.testing.assert_allclose(padded_sequence0.g.flatten() / 2,
padded_sequence1.d.flatten(
)[:np.prod(padded_sequence0.shape)],
atol=1e-4,
err_msg="{} test (w/ accum) with long sequence failed.".format(func_name))
def test_one_hot_forward(seed, inshape, shape, ctx, func_name):
rng = np.random.RandomState(seed)
# Input
input = rng.randint(0, shape[0], size=inshape)
vinput = nn.Variable(input.shape, need_grad=False)
vinput.d = input
with nn.context_scope(ctx), nn.auto_forward():
o = F.one_hot(vinput, shape)
r = ref_one_hot(input, shape)
assert np.allclose(o.d, r)
assert func_name == o.parent.name
def test_large_transform_binary(fname, ctx, func_name):
if not func_name.endswith('Cuda'):
pytest.skip('Grid-strided loop is tested only for CUDA backend')
with nn.context_scope(ctx), nn.auto_forward(True):
a = nn.Variable.from_numpy_array(np.random.randn(
1024, 64, 1)).apply(need_grad=True)
b = nn.Variable.from_numpy_array(np.random.randn(
1024, 64, 3)).apply(need_grad=True)
c = F.mul2(a, b)
c.backward()
def test_random_shift_forward_backward(seed, inshape, shifts, border_mode, ctx,
func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_shift(i, shifts, border_mode, 0, seed)
result_shifts = (0, 0, 0)
max_correl = 0
for shift_amount in itertools.product(*map(
tuple,
map(lambda x: range(*x), [(-2, 3) for _ in range(len(inshape))]))):
r = scipy_shift(inputs[0], shift_amount, mode=border_mode)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
result_shifts = shift_amount
max_correl = correl_and_p[0]
ref = scipy_shift(inputs[0], result_shifts, mode=border_mode)
if shifts is None:
shifts = (0, ) * len(inputs[0].shape)
for result, shift_range in zip(result_shifts, shifts):
assert abs(result) <= shift_range
assert np.allclose(o.d, ref)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
assert np.all(i.g == 0)
def test_one_hot_forward(seed, inshape, shape, ctx, func_name):
rng = np.random.RandomState(seed)
# Input
input = rng.randint(0, shape[0], size=inshape)
vinput = nn.Variable(input.shape, need_grad=False)
vinput.d = input
with nn.context_scope(ctx), nn.auto_forward():
o = F.one_hot(vinput, shape)
r = ref_one_hot(input, shape)
assert np.allclose(o.d, r)
assert func_name == o.parent.name
def sr_loss_with_uncertainty(ctx, pred0, pred1, log_v0, log_v1,
log_s0, log_s1):
v0 = F.exp(log_v0)
v1 = F.exp(log_v1)
squared_error = F.squared_error(pred0, pred1)
s0 = F.exp(log_s0)
s1 = F.exp(log_s1)
with nn.context_scope(ctx):
error = squared_error * (1 / v0 + 1 / v1) + (v0 / v1 + v1 / v0) + (s0 / s1 + s1 / s0)
loss_sr = F.mean(error) * 0.5
return loss_sr
def sr_loss_with_uncertainty(ctx, pred0, pred1, log_v0, log_v1, log_s0,
log_s1):
v0 = F.exp(log_v0)
v1 = F.exp(log_v1)
squared_error = F.squared_error(pred0, pred1)
s0 = F.exp(log_s0)
s1 = F.exp(log_s1)
with nn.context_scope(ctx):
error = squared_error * (1 / v0 + 1 / v1) + (v0 / v1 + v1 / v0) + (
s0 / s1 + s1 / s0)
loss_sr = F.mean(error) * 0.5
return loss_sr
def test_image_augmentation_forward(seed, shape, ctx, func_name):
rng = np.random.RandomState(seed)
inputs = [rng.randn(16, 3, 8, 8).astype(np.float32)]
i = nn.Variable(inputs[0].shape)
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i)
assert o.d.shape == inputs[0].shape
func_kargs = {
'shape': shape,
'pad': (2, 2),
'min_scale': 0.8,
'max_scale': 1.2,
'angle': 0.2,
'aspect_ratio': 1.1,
'distortion': 0.1,
'flip_lr': True,
'flip_ud': False,
'brightness': 0.1,
'brightness_each': True,
'contrast': 1.1,
'contrast_center': 0.5,
'contrast_each': True,
'noise': 0.1,
'seed': 0}
with nn.context_scope(ctx), nn.auto_forward():
o = F.image_augmentation(i, **func_kargs)
assert o.d.shape == (inputs[0].shape[0],) + shape
# Checking recomputation
from nbla_test_utils import recomputation_test
recomputation_test(rng=rng, func=F.image_augmentation, vinputs=[i],
func_args=[], func_kwargs=func_kargs, ctx=ctx)
func_kargs['seed'] = -1
recomputation_test(rng=rng, func=F.image_augmentation, vinputs=[i],
func_args=[], func_kwargs=func_kargs, ctx=ctx)
def solver_tester(rng,
solver,
ref_solver,
solver_args=[],
solver_kwargs={},
num_itr=5,
decay=1e-4,
atol=1e-6,
ctx=None,
solver_name=None):
if ctx is None:
ctx = nn.Context()
# Create params
p1 = nn.Variable([2, 3, 4])
p2 = nn.Variable([3, 4, 1, 2])
p3 = nn.Variable([])
params = OrderedDict([('zZzZ', p1), ('bbb', p2), ('asdfadfdasd', p3)])
for p in params.values():
p.d = rng.randn(*p.shape)
p.g = rng.randn(*p.shape)
with nn.context_scope(ctx):
s = solver(*solver_args, **solver_kwargs)
s.set_parameters(params)
if solver_name is not None:
assert s.name == solver_name
ref_s = ref_solver(*solver_args, **solver_kwargs)
ref_s.set_parameters(params)
# Check weight decay.
grad_copy = OrderedDict([(k, p.g.copy()) for k, p in iteritems(params)])
s.weight_decay(decay)
ref_s.weight_decay(grad_copy, decay)
for p, ref_p in zip(params.values(), grad_copy.values()):
assert np.allclose(ref_p, p.g, atol=atol)
# Check solver udpate.
for i in range(num_itr):
grads = OrderedDict([(k, rng.randn(*p.shape))
for k, p in iteritems(params)])
for k, g in iteritems(grads):
params[k].g = g
s.update()
ref_s.update(grads)
for p, ref_p in zip(params.values(), ref_s.params.values()):
assert np.allclose(ref_p, p.d, atol=atol)
# Check if remove_state_impl work correctly.
s.clear_parameters()
def test_gru_double_backward(seed, num_layers, dropout, bidirectional,
training, seq_len, batch_size, input_size,
hidden_size, with_bias, ctx, func_name):
from nbla_test_utils import backward_function_tester
with nn.context_scope(ctx):
rng = np.random.RandomState(seed)
num_directions = 1
if bidirectional:
num_directions = 2
inputs = [
rng.randn(seq_len, batch_size, input_size).astype(np.float32) * 0.1
]
inputs += [
rng.randn(num_layers, num_directions, batch_size,
hidden_size).astype(np.float32)
]
inputs += [
rng.randn(num_directions, 3, hidden_size, input_size + hidden_size)
]
if num_layers > 1:
inputs += [
rng.randn(max(1, num_layers - 1), num_directions, 3,
hidden_size, num_directions * hidden_size +
hidden_size).astype(np.float32)
]
else:
inputs += [None]
if with_bias:
inputs += [
rng.randn(num_layers, num_directions, 4,
hidden_size).astype(np.float32)
]
else:
inputs += [None]
backward = [False for _ in inputs]
if training:
backward = [True for _ in inputs]
backward_function_tester(rng,
F.gru,
inputs,
func_kwargs=dict(num_layers=num_layers,
dropout=dropout,
bidirectional=bidirectional,
training=training),
atol_f=1e-6,
dstep=1e-3,
backward=backward,
ctx=ctx,
skip_backward_check=True)
def test_random_choice_without_replacement(ctx, func_name, seed):
x = nn.Variable.from_numpy_array(np.array([0, 1, 2]).astype(np.int32))
w = nn.Variable.from_numpy_array(np.array([5, 5, 90]).astype(np.int32))
x.need_grad = True
w.need_grad = True
repeats = 1000
with nn.context_scope(ctx):
y = F.random_choice(x, w, shape=[w.size], replace=False, seed=seed)
r = np.zeros((repeats, w.size)).astype(np.int32)
for i in range(repeats):
y.forward()
r[i] = y.d
assert np.all(np.bincount(r.flatten()) == x.size * [repeats])
def test_copy_from():
shape = [2, 3, 4]
src = nn.NdArray(shape)
dst = nn.NdArray(shape)
src.data = 0
src.cast(dtype=np.uint8)
dst.copy_from(src, use_current_context=False)
assert dst.dtype == np.uint8
from nnabla.ext_utils import get_extension_context
with nn.context_scope(get_extension_context('cpu', dtype='float')):
dst.copy_from(src, use_current_context=True)
assert dst.dtype == np.float32
def test_batch_normalization_forward_backward(seed, axis, decay_rate, eps,
output_stat, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = list(create_inputs(rng, axis))
axes = [axis]
batch_stat = True
function_tester(rng, F.batch_normalization, ref_batch_normalization,
inputs,
func_args=[axes, decay_rate, eps, batch_stat, output_stat],
backward=[True, True, True, False, False],
ctx=ctx, func_name=func_name, dstep=1e-2, atol_b=1e-2)
# Check if running mean and var works.
vinputs = []
for i in inputs:
vinputs.append(nn.Variable(i.shape, True))
vinputs[-1].d = i
for i in range(5):
inputs[0] = rng.randn(*inputs[0].shape)
vinputs[0].d[...] = inputs[0]
ref_y = ref_batch_normalization(
*(inputs + [axes, decay_rate, eps, batch_stat, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.batch_normalization(
*(vinputs + [axes, decay_rate, eps, batch_stat, output_stat]))
assert np.allclose(vinputs[3].d, inputs[3])
assert np.allclose(vinputs[4].d, inputs[4], atol=1e-3)
# Check if global stat mode works
batch_stat = False
if output_stat:
return
ref_y = ref_batch_normalization(
*(inputs + [axes, decay_rate, eps, batch_stat, output_stat]))
with nn.context_scope(ctx), nn.auto_forward():
y = F.batch_normalization(
*(vinputs + [axes, decay_rate, eps, batch_stat, output_stat]))
assert np.allclose(ref_y, y.d, atol=1e-6)
def test_random_crop_forward_backward(seed, inshape, shape, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_crop(i, shape, 0, seed)
if shape is not None:
max_correl = 0
possible_crop_range = [
input - output for output, input in zip(shape, inshape)]
for crop_pos in itertools.product(*map(tuple, map(lambda x: range(*x), [(0, r + 1) for r in possible_crop_range]))):
r = inputs[0][crop_pos[0]:crop_pos[0] + shape[0], crop_pos[1]:crop_pos[1] + shape[1], crop_pos[2]:crop_pos[2] + shape[2]]
assert(o.d.shape == r.shape)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
max_correl = correl_and_p[0]
else:
max_correl = pearsonr(o.d.flatten(), inputs[0].flatten())[0]
assert(max_correl == 1.0)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_diff = rng.randn(*o.shape).astype(i.d.dtype)
o.backward(o_diff)
assert np.all(i.g == 0)
def test_random_shift_forward_backward(seed, inshape, shifts, border_mode, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_shift(i, shifts, border_mode, 0, seed)
result_shifts = (0, 0, 0)
max_correl = 0
for shift_amount in itertools.product(*map(tuple, map(lambda x: range(*x), [(-2, 3) for _ in range(len(inshape))]))):
r = scipy_shift(inputs[0], shift_amount, mode=border_mode)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
result_shifts = shift_amount
max_correl = correl_and_p[0]
ref = scipy_shift(inputs[0], result_shifts, mode=border_mode)
if shifts is None:
shifts = (0,) * len(inputs[0].shape)
for result, shift_range in zip(result_shifts, shifts):
assert abs(result) <= shift_range
assert np.allclose(o.d, ref)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
assert np.all(i.g == 0)
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
h_branch = h
# Convblock 3
h = conv_unit(h_branch, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u0 = conv_unit(h_branch, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u0 = F.average_pooling(u0, (6, 6))
with nn.parameter_scope("u0bn"):
u0 = PF.batch_normalization(u0, batch_stat=not test)
log_var = F.reshape(u0, (u0.shape[0], np.prod(u0.shape[1:])))
# Uncertainty for uncertainty
u1 = conv_unit(h_branch, "u1", 10, k=1, s=1, p=0, act=act, test=test)
u1 = F.average_pooling(u1, (6, 6))
with nn.parameter_scope("u1bn"):
u1 = PF.batch_normalization(u1, batch_stat=not test)
log_s = F.reshape(u1, (u1.shape[0], np.prod(u1.shape[1:])))
return pred, log_var, log_s
def cnn_model_003(ctx, h, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
if not test:
b, c, s, s = h.shape
h = F.image_augmentation(h, (c, s, s),
min_scale=1.0, max_scale=1.5,
angle=0.5, aspect_ratio=1.3, distortion=0.2,
flip_lr=True)
# Convblock0
h = conv_unit(h, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 32 -> 16
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 16 -> 8
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 8 -> 6
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (6, 6))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (6, 6))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def test_unlinked():
v = nn.Variable([2, 3, 4], need_grad=True)
grad = np.random.randn(*v.shape).astype(np.float32)
v.g = grad
v.d = np.random.randn(*v.shape)
import nnabla.functions as F
with nn.context_scope(nn.Context()), nn.auto_forward():
v2 = F.identity(v)
v2_u = v2.unlinked()
v3 = F.identity(v2_u)
v2_u.grad.zero()
v2_g = v2_u.g.copy()
v3.backward(clear_buffer=False)
assert type(v2_u) == type(v2)
assert np.all(v.g == grad)
assert np.all(v2_u.g == v2.g)
assert np.all(v2_u.g == v2_g + 1)
def test_rehape():
v = nn.Variable([2, 3, 4], need_grad=True)
grad = np.random.randn(*v.shape).astype(np.float32)
v.g = grad
v.d = np.random.randn(*v.shape)
import nnabla.functions as F
with nn.context_scope(nn.Context()), nn.auto_forward():
v2 = F.identity(v)
v2_s = v2.reshape((3, 4, 2))
v3 = F.identity(v2_s)
v3.backward(clear_buffer=False)
assert np.all(v2_s.g.flat == v2.g.flat)
assert np.all(v2_s.g == 1)
v2.d = 1
assert np.all(v2_s.d == 1)
v2.g = 1.5
assert np.all(v2_s.g == 1.5)
def test_forward_backward():
batch_size, m, h, w = 4, 3, 32, 32
extension_module = "cpu"
device_id = 0
ctx = extension_context(extension_module, device_id=device_id)
x_l_data = np.random.randn(batch_size, m, h, w)
y_l_data = (np.random.rand(batch_size, 1) * 10).astype(np.int32)
x_l = nn.Variable(x_l_data.shape)
y_l = nn.Variable(y_l_data.shape)
x_l.d = x_l_data
y_l.d = y_l_data
pred = cnn_model_003(ctx, x_l)
with nn.context_scope(ctx):
loss = F.mean(F.softmax_cross_entropy(pred, y_l))
loss.forward()
loss.backward()
def sr_loss_with_uncertainty_and_coef(ctx, pred0, pred1, log_var0, log_var1):
c0 = srwu_learned_coef(ctx, log_var0)
c1 = srwu_learned_coef(ctx, log_var1)
sc0 = sigmas_learned_coef(ctx, log_var0, log_var1)
sc1 = sigmas_learned_coef(ctx, log_var1, log_var0)
c0.need_grad = False
c1.need_grad = False
sc0.need_grad = False
sc1.need_grad = False
#TODO: squared error/absolute error
s0 = F.exp(log_var0)
s1 = F.exp(log_var1)
squared_error = F.squared_error(pred0, pred1)
with nn.context_scope(ctx):
loss_sr = F.mean(
squared_error * (c0 / s0 + c1 / s1) + (sc0 * s0 / s1 + sc1 * s1 / s0)) * 0.5
return loss_sr
def _setup(self, delete=True):
"""Create a function instance and execute setup.
Args:
delete (bool): Delete buffered variables.
"""
if delete:
self.clear()
with nn.context_scope(self.ctx):
outputs = self.func(
*(self.inputs_f + self.func_args), **self.func_kwargs)
if not hasattr(outputs, '__iter__'):
self.outputs = [outputs]
else:
self.outputs = outputs
self.func_ins = self.outputs[0].parent
self.inputs = self.func_ins.inputs
def test_random_flip_forward_backward(seed, axes, ctx, func_name):
from nbla_test_utils import cap_ignore_region, function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(2, 3, 4).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_flip(i, axes, 0, seed)
flip_close = np.allclose(o.d, ref_flip(inputs[0], axes))
assert flip_close or (not flip_close and np.allclose(o.d, i.d))
assert o.parent.name == func_name
# NNabla backward
orig_grad = rng.randn(*i.shape).astype(i.data.dtype)
i.g[...] = orig_grad
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.g = o_grad
o.parent.backward([i], [o])
# Verify
if flip_close:
ref_grad = ref_flip(o_grad, axes)
else:
ref_grad = o_grad
assert np.allclose(i.g, orig_grad + ref_grad)
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad)
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o.backward(o_grad)
assert np.all(i.g == 0)
def cnn_model_003(ctx, x, act=F.elu, do=True, test=False):
with nn.context_scope(ctx):
# Convblock0
h = conv_unit(x, "conv00", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv01", 128, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv02", 128, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 28 -> 14
with nn.parameter_scope("bn0"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 1
h = conv_unit(h, "conv10", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv11", 256, k=3, s=1, p=1, act=act, test=test)
h = conv_unit(h, "conv12", 256, k=3, s=1, p=1, act=act, test=test)
h = F.max_pooling(h, (2, 2)) # 14 -> 7
with nn.parameter_scope("bn1"):
h = PF.batch_normalization(h, batch_stat=not test)
if not test and do:
h = F.dropout(h)
# Convblock 2
h = conv_unit(h, "conv20", 512, k=3, s=1, p=0, act=act, test=test) # 7 -> 5
h = conv_unit(h, "conv21", 256, k=1, s=1, p=0, act=act, test=test)
h = conv_unit(h, "conv22", 128, k=1, s=1, p=0, act=act, test=test)
u = h
# Convblock 3
h = conv_unit(h, "conv23", 10, k=1, s=1, p=0, act=act, test=test)
h = F.average_pooling(h, (5, 5))
with nn.parameter_scope("bn2"):
h = PF.batch_normalization(h, batch_stat=not test)
pred = F.reshape(h, (h.shape[0], np.prod(h.shape[1:])))
# Uncertainty
u = conv_unit(u, "u0", 10, k=1, s=1, p=0, act=act, test=test)
u = F.average_pooling(u, (5, 5))
with nn.parameter_scope("u0bn"):
u = PF.batch_normalization(u, batch_stat=not test)
log_var = F.reshape(u, (u.shape[0], np.prod(u.shape[1:])))
return pred, log_var
def test_dropout_forward_backward(p, seed, ctx, func_name):
from nbla_test_utils import cap_ignore_region, function_tester
rng = np.random.RandomState(seed)
inputs = [
cap_ignore_region(
rng.randn(2, 3, 4).astype(np.float32) * 2,
(-1e-3, 1e-3))] # Ensure there is no zero.
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.dropout(i, p)
scale = 1. / (1. - p)
mask = o.d != 0
assert np.allclose(o.d, i.d * mask * scale)
assert o.parent.name == func_name
# NNabla backward
orig_grad = rng.randn(*i.shape).astype(i.data.dtype)
i.g[...] = orig_grad
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
ref_grad = o_grad * mask * scale
# Verify
assert np.allclose(i.g, orig_grad + ref_grad)
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad)
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o.backward(o_grad)
assert np.all(i.g == 0)