def test_2layer_deconv(deconv_n4_hw4_c1_5x5):
cf1 = ConvParams(**deconv_n4_hw4_c1_5x5)
# 2nd layer filter
fshape2 = (4, 3, 3)
str_h = str_w = 2
K, R, S = fshape2
C, D, H, W, N = cf1.dimO # 2nd layer input dimensions
cf2 = ConvParams(C=C, D=D, H=H, W=W, N=N, K=K, R=R, S=S, str_h=str_h, str_w=str_w, deconv=True)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf1.ax_i)
filter1_value = rng.uniform(-0.5, 0.5, cf1.ax_f)
filter2_value = rng.uniform(-0.5, 0.5, cf2.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf2.ax_o)
inputs = ng.placeholder(cf1.ax_i)
filters1 = ng.placeholder(cf1.ax_f)
filters2 = ng.placeholder(cf2.ax_f)
errors = ng.placeholder(cf2.ax_o)
out1 = ng.deconvolution(cf1.conv_params, inputs, filters1, axes=cf1.ax_o)
out2 = ng.deconvolution(cf2.conv_params, out1, filters2, axes=cf2.ax_o)
bprop_out = ng.deriv(out2, inputs, errors)
updat_out2 = ng.deriv(out2, filters2, errors)
updat_out1 = ng.deriv(out2, filters1, errors)
with executor([out1, out2, bprop_out, updat_out1, updat_out2],
inputs, filters1, filters2, errors) as conv_executor:
out1_ng, out2_ng, gradI_ng, gradF1_ng, gradF2_ng = \
conv_executor(input_value, filter1_value, filter2_value, error_value)
# Compute reference with NumPy
# fprop
out1_np = reference_deconv_fprop(cf1.conv_params, input_value, filter1_value)
out2_np = reference_deconv_fprop(cf2.conv_params, out1_np, filter2_value)
# bprop
gradI2_np, gradF2_np = reference_deconv_bprop(cf2.conv_params,
error_value,
out1_np,
filter2_value)
gradI1_np, gradF1_np = reference_deconv_bprop(cf1.conv_params,
gradI2_np,
input_value,
filter1_value)
# Compare fprop
assert np.allclose(out1_ng, out1_np, rtol=0.01, atol=0)
assert np.allclose(out2_ng, out2_np, rtol=0.01, atol=0)
# Compare bprop
assert np.allclose(gradI_ng, gradI1_np, rtol=0.01, atol=0)
# Compare update
assert np.allclose(gradF1_ng, gradF1_np, rtol=0.01, atol=0)
assert np.allclose(gradF2_ng, gradF2_np, rtol=0.01, atol=0)
def test_conv(n64_hw32_c32_3x3):
cf = ConvParams(**n64_hw32_c32_3x3)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf.ax_i)
filter_value = rng.uniform(-0.5, 0.5, cf.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
errors = ng.placeholder(cf.ax_o)
output = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
bprop_out = bprop_conv(errors, inputs, filters, output)
updat_out = update_conv(errors, inputs, filters, output)
with executor([output, bprop_out, updat_out], inputs, filters, errors) as conv_executor:
result_ng, gradI_ng, gradF_ng = conv_executor(input_value, filter_value, error_value)
# Compute reference with NumPy
result_np, gradI_np, gradF_np = reference_conv(cf.dimI, cf.dimF, cf.dimO,
cf.conv_params,
input_value, filter_value, error_value)
# Compare fprop
assert np.allclose(result_ng, result_np, rtol=0, atol=0.5)
# Compare bprop
assert np.allclose(gradI_ng, gradI_np, rtol=0, atol=0.5)
# Compare update
assert np.allclose(gradF_ng, gradF_np, rtol=0, atol=2)
def test_conv_flatten_deriv(n4_hw12_c3_5x5):
"""
Test deriv of conv followed by flatten
"""
cf = ConvParams(**n4_hw12_c3_5x5)
axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]])
axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length)
for ax in axes_rsck])
axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]])
# broadcast input / filter axes
input_var = ng.variable(cf.ax_i).named('input')
input_val = np.ones(input_var.axes.lengths)
filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter')
filter_var = filter_rsck_prime
filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck')
filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck')
filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk')
# convolution
output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o)
output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk)
# slice away the oD
out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)]
output_npqk = ng.tensor_slice(output_nmpqk, out_slicing)
output = ng.flatten_at(output_npqk, idx=1)
# cost and grad
cost = ng.sum(output, out_axes=())
filter_val = np.ones(filter_var.axes.lengths)
with ExecutorFactory() as factory:
conv_comp = factory.executor(output, filter_var, input_var)
grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var)
grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var)
grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var)
grad_input_sym_comp = factory.derivative(cost, input_var, filter_var)
conv_val = conv_comp(filter_val, input_val)
conv_val_num = np.empty_like(conv_val)
conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1]))
ng.testing.assert_allclose(conv_val, conv_val_num)
grad_filter_num_val = grad_filter_num_comp(filter_val, input_val)
grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val)
ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val)
grad_input_num_val = grad_input_num_comp(input_val, filter_val)
grad_input_sym_val = grad_input_sym_comp(input_val, filter_val)
ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def execute_convolution(image_height,
image_width,
filter_height,
filter_width,
channel=16,
batch_size=32,
filter_count=8,
image_3rd_dim=1,
filter_3rd_dim=1,
padding=(0, 0, 0),
stride=(1, 1, 1),
dilation=1,
np_comparison=False):
pad_h, pad_w, pad_d = padding
str_h, str_w, str_d = stride
cf = ConvParams(C=channel,
N=batch_size,
K=filter_count,
D=image_3rd_dim,
H=image_height,
W=image_width,
T=filter_3rd_dim,
R=filter_height,
S=filter_width,
pad_d=pad_d,
pad_h=pad_h,
pad_w=pad_w,
str_d=str_d,
str_h=str_h,
str_w=str_w,
dil_d=dilation,
dil_h=dilation,
dil_w=dilation)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
rng = RandomTensorGenerator(0, np.float32)
input_value = rng.uniform(-4, 4, cf.ax_i, dtype=int)
filter_value = rng.uniform(-4, 4, cf.ax_f, dtype=int)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
with executor(
ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o),
inputs, filters) as const_executor:
out = const_executor(input_value, filter_value)
if np_comparison:
np_out, gradInp, gradF_np = \
reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value,
error_value)
return out, np_out
return out
def test_wrong_input_shape_length():
"""
test wrong input shape length
"""
cf = ConvParams()
ax_i = cf.ax_i[:-1]
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(cf.ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(cf.conv_params, inputs, filters, {})
assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\
.format(len(ax_i))
def test_first_axes_not_same():
"""
test first axes are not the same
"""
cf = ConvParams()
ax_i = cf.ax_i[1:2] + cf.ax_i[0:1] + cf.ax_i[2:] # D, C, H, W, N
inputs = ng.placeholder(ax_i)
filters = ng.placeholder(cf.ax_f)
with pytest.raises(ValueError) as exinfo:
ng.convolution(cf.conv_params, inputs, filters, {})
assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \
'are not the same.'.format(
inputs=inputs.axes[0],
filters=filters.axes[0])
def test_convolution_backprop(n128_hw32_c3_2x2):
"""
test convolution backprop path
"""
cf = ConvParams(**n128_hw32_c3_2x2)
inputs = ng.placeholder(axes=cf.ax_i)
filters = ng.placeholder(axes=cf.ax_f)
# randomly initialize
input_value = rng.uniform(-1, 1, cf.ax_i)
filter_value = rng.uniform(-1, 1, cf.ax_f)
output = ng.sum(ng.convolution(cf.conv_params, inputs, filters, cf.ax_o), out_axes=())
with ExecutorFactory() as factory:
dcdf_sym_fun = factory.derivative(output, filters, inputs)
dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs)
dcdf_sym_val = dcdf_sym_fun(filter_value, input_value)
dcdf_num_val = dcdf_num_fun(filter_value, input_value)
ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=0.01)
def test_deconv(transformer_factory, deconv_n4_hw4_c1_5x5):
cf = ConvParams(**deconv_n4_hw4_c1_5x5)
# randomly initialize
input_value = rng.uniform(-0.5, 0.5, cf.ax_i)
filter_value = rng.uniform(-0.5, 0.5, cf.ax_f)
error_value = rng.uniform(-0.5, 0.5, cf.ax_o)
inputs = ng.placeholder(cf.ax_i)
filters = ng.placeholder(cf.ax_f)
errors = ng.placeholder(cf.ax_o)
output = ng.deconvolution(cf.conv_params, inputs, filters, axes=cf.ax_o)
bprop_out = ng.deriv(output, inputs, errors)
updat_out = ng.deriv(output, filters, errors)
with executor([output, bprop_out, updat_out], inputs, filters,
errors) as conv_executor:
result_ng, gradI_ng, gradF_ng = conv_executor(input_value,
filter_value,
error_value)
# Compute reference with NumPy
result_np = reference_deconv_fprop(cf.conv_params, input_value,
filter_value)
gradI_np, gradF_np = reference_deconv_bprop(cf.conv_params, error_value,
input_value, filter_value)
# Compare fprop
assert np.allclose(result_ng, result_np, rtol=0.1, atol=0)
# Compare bprop
assert np.allclose(gradI_ng, gradI_np, rtol=0.1, atol=0)
# Compare update
assert np.allclose(gradF_ng, gradF_np, rtol=0.1, atol=0)