def ref_deconvolution_2d(x,
w,
b,
base_axis,
pad,
stride,
dilation,
group,
channel_last=False,
output_padding=(0, 0)):
if channel_last:
transpose_x = refs.ChannelLastToFirstTranspose(x.ndim, len(pad))
transpose_w = refs.ChannelLastToFirstTranspose(w.ndim, len(pad))
return transpose_x.inv(
ref_deconvolution_2d(transpose_x(x), transpose_w(w), b, base_axis,
pad, stride, dilation, group, False,
output_padding))
y = []
for xx in x.reshape((-1, ) + x.shape[base_axis:]):
y += [
refs.deconvolution_2d(xx,
w,
b,
pad,
stride,
dilation,
group,
output_padding=output_padding)[np.newaxis]
]
y = np.vstack(y)
return y.reshape(x.shape[:base_axis] + y.shape[1:])
def test_deconvolution_2d_forward_backward(inshape, kernel, outmaps, pad,
stride, dilation, group, with_bias,
channel_last, output_padding,
seed, ctx, func_name):
from nbla_test_utils import function_tester
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('channel_last=True is only supported in CUDNN backend.')
base_axis = len(inshape) - len(kernel) - 1
inmaps = inshape[base_axis]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
i = rng.randn(*inshape).astype(np.float32)
kshape = (inmaps,) + (outmaps // group,) + kernel
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
k = rng.randn(*kshape).astype(np.float32)
base_axis = len(inshape) - 3
b = None
if with_bias:
b = rng.randn(outmaps).astype(np.float32)
inputs = [i, k, b]
func_args = [base_axis, pad, stride, dilation, group, channel_last,
output_padding]
function_tester(rng, F.deconvolution, ref_deconvolution_2d, inputs,
func_args=func_args, func_name=func_name, ctx=ctx,
atol_f=1e-4, atol_b=1e-2, atol_accum=1e-5, dstep=1e-2)
def core_test_convolution_forward_backward(inshape, kernel, outmaps, pad,
stride, dilation, group,
channel_last, with_bias, seed, ctx,
func_name):
from nbla_test_utils import function_tester
if func_name == 'ConvolutionCuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip(
'channel_last=True is only supported in CUDNN backend so far.')
if channel_last and func_name.endswith('Cudnn') and (
np.any(np.asarray(dilation) > 1) or group > 1):
import nnabla_ext.cuda as nc
major, minor, revision = map(int, nc.__cudnn_version__.split('.'))
version = major * 1000 + minor * 100
if version < 7200:
pytest.skip(
'channel_last dilated convolution not work in CUDNN {}.'.
format(version))
base_axis = len(inshape) - len(kernel) - 1
inmaps = inshape[base_axis]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
i = rng.randn(*inshape).astype(np.float32)
kshape = (outmaps, ) + (inmaps // group, ) + kernel
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
k = rng.randn(*kshape).astype(np.float32)
b = None
if with_bias:
b = rng.randn(outmaps).astype(np.float32)
inputs = [i, k, b]
atol_half = 1.0 if inmaps > 64 else 1e-1
function_tester(
rng,
F.convolution,
ref_convolution,
inputs,
func_args=[base_axis, pad, stride, dilation, group, channel_last],
atol_f=1e-4,
atol_b=1e-2,
atol_accum=1e-5,
dstep=1e-2,
ctx=ctx,
func_name=func_name,
atol_half=atol_half)
def ref_convolution(x, w, b, base_axis, pad, stride, dilation, group, channel_last):
if channel_last:
t = refs.ChannelLastToFirstTranspose(x.ndim, len(pad))
x = t(x)
tw = refs.ChannelLastToFirstTranspose(w.ndim, len(pad))
w = tw(w)
y = ref_convolution(x, w, b, base_axis, pad,
stride, dilation, group, False)
return t.inv(y)
y = []
for xx in x.reshape((-1,) + x.shape[base_axis:]):
y += [refs.convolution_nd(xx, w, b, pad, stride,
dilation, group)[np.newaxis]]
y = np.vstack(y)
return y.reshape(x.shape[:base_axis] + y.shape[1:])
def test_sum_pooling_3d_double_backward(seed, inshape, kernel, stride, pad, ignore_border, channel_last,
ctx, func_name):
from nbla_test_utils import backward_function_tester, grad_function_forward_function_output
from nnabla.backward_function.sum_pooling import SumPoolingDataGrad
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('Channel last is only supported in Cudnn so far')
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
if not ignore_border and func_name.endswith('Cudnn'):
pytest.skip('ignore_border=False in Cudnn is not supported.')
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
func_args = [kernel, stride, ignore_border, pad, channel_last]
# 2nd-order
backward_function_tester(rng, F.sum_pooling, inputs=inputs,
func_args=func_args, ctx=ctx)
# 3rd-order
df, y = grad_function_forward_function_output(SumPoolingDataGrad,
F.sum_pooling,
ctx, inputs,
*func_args)
df.xshape = inputs[0].shape
ginputs = [rng.randn(*y.shape)]
backward_function_tester(rng, df,
inputs=ginputs, ctx=ctx, atol_accum=3e-2, non_accum_check=True)
def test_max_pooling_3d_double_backward(seed, inshape, kernel, stride, pad,
ignore_border, channel_last, ctx,
func_name):
# pytest.skip('`>3`-dimension are not supported.')
from nbla_test_utils import backward_function_tester, cap_ignore_region
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('Channel last is only supported in Cudnn so far')
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
func_args = [kernel, stride, ignore_border, pad, channel_last]
# 2nd-order
backward_function_tester(rng,
F.max_pooling,
inputs=inputs,
func_args=func_args,
ctx=ctx)
# 3nd-order
import nnabla as nn
y = F.max_pooling(nn.Variable(inputs[0].shape), *func_args)
ginputs = [rng.randn(*y.shape), inputs[0]]
backward_function_tester(rng,
F.max_pooling_backward,
inputs=ginputs,
func_args=func_args,
ctx=ctx,
backward=[True, False],
non_accum_check=True)
def test_average_pooling_2d_double_backward(seed, inshape, kernel, stride, pad, ignore_border,
channel_last,
including_pad, ctx, func_name):
from nbla_test_utils import backward_function_tester, grad_function_forward_function_output
from nnabla.backward_function.average_pooling import AveragePoolingDataGrad
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('Channel last is only supported in Cudnn so far')
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
func_args = [kernel, stride, ignore_border,
pad, channel_last, including_pad]
# 2nd-order
backward_function_tester(rng, F.average_pooling,
inputs=inputs, func_args=func_args,
ctx=ctx)
# 3rd-order
average_pooling_data_grad, y = grad_function_forward_function_output(AveragePoolingDataGrad,
F.average_pooling,
ctx, inputs,
*func_args)
average_pooling_data_grad.xshape = inputs[0].shape
ginputs = [rng.randn(*y.shape)]
backward_function_tester(rng, average_pooling_data_grad,
inputs=ginputs, func_args=[],
ctx=ctx)
def ref_unpooling(x, kernel, channel_last):
if channel_last:
n_sdim = len(kernel)
t = refs.ChannelLastToFirstTranspose(x.ndim, n_sdim)
x = t(x)
y = x
for ind, p in enumerate(kernel[::-1]):
y = y.repeat(p, axis=y.ndim - (ind + 1))
if channel_last:
y = t.inv(y)
return y
def ref_sum_pooling(x, kernel, stride, ignore_border, pad, channel_last):
if channel_last:
t = refs.ChannelLastToFirstTranspose(x.ndim, len(kernel))
x = t(x)
y = ref_sum_pooling(x, kernel, stride, ignore_border, pad, False)
return t.inv(y)
if len(kernel) == 3:
y = ref_sum_pooling_3d(x, kernel, stride, ignore_border, pad)
return y
y = ref_sum_pooling_2d(x, kernel, stride, ignore_border, pad)
return y
def test_sum_pooling_3d(seed, inshape, kernel, stride, pad, ignore_border, channel_last,
ctx, func_name):
from nbla_test_utils import function_tester
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('Channel last is only supported in Cudnn so far')
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
func_args = [kernel, stride, ignore_border, pad, channel_last]
function_tester(rng, F.sum_pooling, ref_sum_pooling, inputs=inputs,
func_args=func_args, func_name=func_name, ctx=ctx,
atol_f=1e-6, atol_b=1e-2)
def ref_average_pooling(x, kernel, stride, ignore_border, pad, channel_last, including_pad):
if channel_last:
t = refs.ChannelLastToFirstTranspose(x.ndim, len(kernel))
x = t(x)
y = ref_average_pooling(
x, kernel, stride, ignore_border, pad, False, including_pad)
y = t.inv(y)
return y
if len(kernel) == 3:
y = ref_average_pooling_3d(
x, kernel, stride, ignore_border, pad, including_pad)
return y
y = ref_average_pooling_2d(
x, kernel, stride, ignore_border, pad, including_pad)
return y
def ref_interpolate(x,
scale,
output_size,
mode,
align_corners=True,
half_pixel=False,
half_pixel_for_nn=False,
channel_last=False):
assert scale or output_size, 'Need either scale or output_size.'
assert not scale or len(scale) in (1, 2, 3), 'Only 1D/2D/3D'
assert not output_size or len(output_size) in (1, 2, 3), 'Only 1D/2D/3D'
if channel_last:
n_sdim = len(scale) if scale else len(output_size)
t = refs.ChannelLastToFirstTranspose(x.ndim, n_sdim)
x = t(x)
if not output_size:
output_size = np.floor(np.array(scale) * x.shape[-len(scale):])
output_size = tuple(map(int, output_size))
if mode == "nearest":
if len(output_size) == 1:
out = ref_nearest_interpolate_1d(x, output_size, align_corners,
half_pixel, half_pixel_for_nn)
out = t.inv(out) if channel_last else out
if len(output_size) == 2:
out = ref_nearest_interpolate_2d(x, output_size, align_corners,
half_pixel, half_pixel_for_nn)
out = t.inv(out) if channel_last else out
if len(output_size) == 3:
out = ref_nearest_interpolate_3d(x, output_size, align_corners,
half_pixel, half_pixel_for_nn)
out = t.inv(out) if channel_last else out
elif mode == "linear":
if len(output_size) == 1:
out = ref_linear_interpolate_1d(x, output_size, align_corners,
half_pixel)
if len(output_size) == 2:
out = ref_linear_interpolate_2d(x, output_size, align_corners,
half_pixel)
if len(output_size) == 3:
out = ref_linear_interpolate_3d(x, output_size, align_corners,
half_pixel)
out = t.inv(out) if channel_last else out
return out
def test_max_pooling_3d_double_backward(seed, inshape, kernel, stride, pad, ignore_border, channel_last,
ctx, func_name):
# pytest.skip('`>3`-dimension are not supported.')
# TODO: some test fail
from nbla_test_utils import backward_function_tester, cap_ignore_region
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('Channel last is only supported in Cudnn so far')
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
func_args = [kernel, stride, ignore_border, pad, channel_last]
backward_function_tester(rng, F.max_pooling, None, inputs=inputs,
func_args=func_args, func_name=func_name, ctx=ctx,
atol_f=1e-2, atol_b=1e-1, atol_accum=1e-1, dstep=1e-3)
def ref_interpolate(x, scale, output_size, mode, align_corners=None, channel_last=False):
assert scale or output_size, 'Need either scale or output_size.'
assert not scale or len(scale) in (1, 2, 3), 'Only 1D/2D/3D'
assert not output_size or len(output_size) in (1, 2, 3), 'Only 1D/2D/3D'
if channel_last:
n_sdim = len(scale) if scale else len(output_size)
t = refs.ChannelLastToFirstTranspose(x.ndim, n_sdim)
x = t(x)
if not output_size:
output_size = np.floor(np.array(scale) * x.shape[-len(scale):])
output_size = tuple(map(int, output_size))
if mode == "nearest":
osize = output_size
isize = x.shape[-len(osize):]
scale = [i / o for i, o in zip(isize, osize)]
index = [s * (np.arange(o) + 0.5) for s, o in zip(scale, osize)]
index = [idx.astype(np.int32) for idx in index]
index = [np.minimum(idx, i - 1) for idx, i in zip(index, isize)]
xx = x.reshape(-1, *isize)
ib = np.arange(xx.shape[0])
yy = xx[np.ix_(ib, *index)]
out = yy.reshape(x.shape[:-len(osize)] + osize)
out = t.inv(out) if channel_last else out
return out
elif mode == "linear":
if len(output_size) == 1:
out = ref_linear_interpolate_1d(
x, output_size, mode, align_corners)
if len(output_size) == 2:
out = ref_linear_interpolate_2d(
x, output_size, mode, align_corners)
if len(output_size) == 3:
out = ref_linear_interpolate_3d(
x, output_size, mode, align_corners)
out = t.inv(out) if channel_last else out
return out
def core_test_convolution_double_backward(inshape,
kernel,
outmaps,
pad,
stride,
dilation,
group,
channel_last,
with_bias,
seed,
ctx,
func_name,
non_accum_check=True,
atol_f=1e-4,
atol_b=1e-3,
atol_accum=8e-2,
dstep=1e-3):
from nbla_test_utils import backward_function_tester, grad_function_forward_function_output
from nnabla.backward_function.convolution import ConvolutionDataGrad, ConvolutionFilterGrad
if func_name == 'ConvolutionCuda':
pytest.skip(
'CUDA Convolution N-D is only supported in CUDNN extension')
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip(
'channel_last=True is only supported in CUDNN backend so far.')
if channel_last and func_name.endswith('Cudnn') and (
np.any(np.asarray(dilation) > 1) or group > 1):
import nnabla_ext.cuda as nc
major, minor, revision = map(int, nc.__cudnn_version__.split('.'))
version = major * 1000 + minor * 100
if version < 7200:
pytest.skip(
'channel_last dilated convolution not work in CUDNN {}.'.
format(version))
base_axis = len(inshape) - len(kernel) - 1
inmaps = inshape[base_axis]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
i = np.clip(rng.randn(*inshape).astype(np.float32), -0.8, 0.8)
kshape = (outmaps, ) + (inmaps // group, ) + kernel
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
k = np.clip(rng.randn(*kshape).astype(np.float32), -0.8, 0.8)
b = None
if with_bias:
b = np.clip(rng.randn(outmaps).astype(np.float32), -0.8, 0.8)
inputs = [i, k, b]
atol_half = 1.0 if inmaps > 64 else 1e-1
func_args = [base_axis, pad, stride, dilation, group, channel_last]
# Convolution
backward_function_tester(rng,
F.convolution,
inputs,
func_args=func_args,
atol_f=atol_f,
atol_accum=atol_accum,
dstep=dstep,
ctx=ctx)
# DataGrad
df, y = grad_function_forward_function_output(ConvolutionDataGrad,
F.convolution, ctx, inputs,
*func_args)
df.xshape = i.shape
ginputs = [rng.randn(*y.shape), k]
backward_function_tester(rng,
df,
ginputs,
func_args=[],
atol_f=atol_f,
atol_b=atol_b,
atol_accum=atol_accum,
dstep=dstep,
ctx=ctx,
non_accum_check=non_accum_check)
# FilterGrad
df, y = grad_function_forward_function_output(ConvolutionFilterGrad,
F.convolution, ctx, inputs,
*func_args)
df.wshape = k.shape
ginputs = [rng.randn(*y.shape), i]
backward_function_tester(rng,
df,
ginputs,
func_args=[],
atol_f=atol_f,
atol_b=atol_b,
atol_accum=atol_accum,
dstep=dstep,
ctx=ctx,
non_accum_check=non_accum_check)
def test_deconvolution_2d_double_backward(inshape, kernel, outmaps, pad,
stride, dilation, group, with_bias,
channel_last, output_padding, seed,
ctx, func_name):
from nbla_test_utils import function_tester, backward_function_tester, grad_function_forward_function_output
from nnabla.backward_function.deconvolution import DeconvolutionDataGrad, DeconvolutionFilterGrad
if channel_last and not func_name.endswith('Cudnn'):
pytest.skip('channel_last=True is only supported in CUDNN backend.')
base_axis = len(inshape) - len(kernel) - 1
inmaps = inshape[base_axis]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
rng = np.random.RandomState(seed)
i = np.clip(rng.randn(*inshape).astype(np.float32), -0.5, 0.5)
kshape = (inmaps, ) + (outmaps // group, ) + kernel
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
k = np.clip(rng.randn(*kshape).astype(np.float32), -0.5, 0.5)
base_axis = len(inshape) - 3
b = None
if with_bias:
b = np.clip(rng.randn(outmaps).astype(np.float32), -0.5, 0.5)
inputs = [i, k, b]
func_args = [
base_axis, pad, stride, dilation, group, channel_last, output_padding
]
# Deconvolution
backward_function_tester(rng,
F.deconvolution,
inputs,
func_args=func_args,
ctx=ctx,
atol_accum=1e-1)
# DataGrad
df, y = grad_function_forward_function_output(DeconvolutionDataGrad,
F.deconvolution, ctx, inputs,
*func_args)
df.xshape = i.shape
ginputs = [rng.randn(*y.shape), k]
backward_function_tester(rng,
df,
ginputs,
ctx=ctx,
atol_accum=1e-1,
non_accum_check=True)
# FilterGrad
df, y = grad_function_forward_function_output(DeconvolutionFilterGrad,
F.deconvolution, ctx, inputs,
*func_args)
df.wshape = k.shape
ginputs = [rng.randn(*y.shape), i]
backward_function_tester(rng,
df,
ginputs,
func_args=[],
ctx=ctx,
atol_accum=1e-1,
non_accum_check=True)
def test_forward_backward_2d(inshape, kernel, out_channels, pad, stride,
dilation, group, deformable_group, with_mask,
channel_last, with_bias, seed, ctx, func_name):
if channel_last:
pytest.skip(
'channel_last=True is not supported in any backends so far.')
import platform
if platform.machine().startswith("arm"):
pytest.skip('Skip the arm platform temporarily.')
rng = np.random.RandomState(seed)
# Create arguments
base_axis = len(inshape) - len(kernel) - 1
func_args = [
base_axis, pad, stride, dilation, group, deformable_group, channel_last
]
# Compute shapes
in_channels = inshape[base_axis]
kshape = (out_channels, in_channels // group) + kernel
offset_channels = 2 * deformable_group * kernel[0] * kernel[1]
offset_shape = inshape[0:base_axis] + \
(offset_channels,) + inshape[base_axis + 1:]
mask_shape = inshape[0:base_axis] + \
(deformable_group * kernel[0] * kernel[1],) + inshape[base_axis + 1:]
if channel_last:
t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel))
inshape = tuple(inshape[i] for i in t.inv_axes)
t = refs.ChannelLastToFirstTranspose(len(offset_shape), len(kernel))
offset_shape = tuple(offset_shape[i] for i in t.inv_axes)
t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel))
kshape = tuple(kshape[i] for i in t.inv_axes)
# Create inputs
x = rng.randn(*inshape).astype(np.float32)
w = rng.randn(*kshape).astype(np.float32)
b = rng.randn(out_channels).astype(np.float32) if with_bias else None
# Because numerical gradient cannot be calculated correctly
# near the input boundary, offsets are generated to avoid this case.
# 1. Generate offsets in [-1.9, 1.9].
offsets = (3.8 * rng.rand(*offset_shape).astype(np.float32)) - 1.9
# 2. Adhoc remove the values dstep-neighborhood of {-1, 0, 1}; selecting bad
# values as 0.1-neighborhood (large enough dstep-neighborhood) and shifting
# them +0.5 (must larger than 2 * dstep).
offsets += np.logical_or(
np.abs(offsets - np.floor(offsets)) < 0.1,
np.abs(offsets - np.ceil(offsets)) < 0.1).astype(np.int) * 0.5
mask = rng.rand(*mask_shape).astype(np.float32) if with_mask else None
inputs = [x, w, offsets, mask, b]
# Test
atol_half = 1.0 if in_channels > 64 else 1.5e-1
function_tester(rng,
F.deformable_convolution,
ref_deformable_convolution_2d,
inputs,
func_args,
atol_f=1e-4,
atol_b=1e-2,
atol_accum=1e-5,
dstep=1e-2,
ctx=ctx,
func_name=func_name,
atol_half=atol_half)
