def test_2layer_deconv(deconv_n4_hw4_c1_5x5): cf1 = ConvParams(**deconv_n4_hw4_c1_5x5) # 2nd layer filter fshape2 = (4, 3, 3) str_h = str_w = 2 K, R, S = fshape2 C, D, H, W, N = cf1.dimO # 2nd layer input dimensions cf2 = ConvParams(C=C, D=D, H=H, W=W, N=N, K=K, R=R, S=S, str_h=str_h, str_w=str_w, deconv=True) # randomly initialize input_value = rng.uniform(-0.5, 0.5, cf1.ax_i) filter1_value = rng.uniform(-0.5, 0.5, cf1.ax_f) filter2_value = rng.uniform(-0.5, 0.5, cf2.ax_f) error_value = rng.uniform(-0.5, 0.5, cf2.ax_o) inputs = ng.placeholder(cf1.ax_i) filters1 = ng.placeholder(cf1.ax_f) filters2 = ng.placeholder(cf2.ax_f) errors = ng.placeholder(cf2.ax_o) out1 = ng.deconvolution(cf1.conv_params, inputs, filters1, axes=cf1.ax_o) out2 = ng.deconvolution(cf2.conv_params, out1, filters2, axes=cf2.ax_o) bprop_out = ng.deriv(out2, inputs, errors) updat_out2 = ng.deriv(out2, filters2, errors) updat_out1 = ng.deriv(out2, filters1, errors) with executor([out1, out2, bprop_out, updat_out1, updat_out2], inputs, filters1, filters2, errors) as conv_executor: out1_ng, out2_ng, gradI_ng, gradF1_ng, gradF2_ng = \ conv_executor(input_value, filter1_value, filter2_value, error_value) # Compute reference with NumPy # fprop out1_np = reference_deconv_fprop(cf1.conv_params, input_value, filter1_value) out2_np = reference_deconv_fprop(cf2.conv_params, out1_np, filter2_value) # bprop gradI2_np, gradF2_np = reference_deconv_bprop(cf2.conv_params, error_value, out1_np, filter2_value) gradI1_np, gradF1_np = reference_deconv_bprop(cf1.conv_params, gradI2_np, input_value, filter1_value) # Compare fprop assert np.allclose(out1_ng, out1_np, rtol=0.01, atol=0) assert np.allclose(out2_ng, out2_np, rtol=0.01, atol=0) # Compare bprop assert np.allclose(gradI_ng, gradI1_np, rtol=0.01, atol=0) # Compare update assert np.allclose(gradF1_ng, gradF1_np, rtol=0.01, atol=0) assert np.allclose(gradF2_ng, gradF2_np, rtol=0.01, atol=0)
def test_conv(n64_hw32_c32_3x3): cf = ConvParams(**n64_hw32_c32_3x3) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize input_value = rng.uniform(-0.5, 0.5, cf.ax_i) filter_value = rng.uniform(-0.5, 0.5, cf.ax_f) error_value = rng.uniform(-0.5, 0.5, cf.ax_o) inputs = ng.placeholder(cf.ax_i) filters = ng.placeholder(cf.ax_f) errors = ng.placeholder(cf.ax_o) output = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o) bprop_out = bprop_conv(errors, inputs, filters, output) updat_out = update_conv(errors, inputs, filters, output) with executor([output, bprop_out, updat_out], inputs, filters, errors) as conv_executor: result_ng, gradI_ng, gradF_ng = conv_executor(input_value, filter_value, error_value) # Compute reference with NumPy result_np, gradI_np, gradF_np = reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value, error_value) # Compare fprop assert np.allclose(result_ng, result_np, rtol=0, atol=0.5) # Compare bprop assert np.allclose(gradI_ng, gradI_np, rtol=0, atol=0.5) # Compare update assert np.allclose(gradF_ng, gradF_np, rtol=0, atol=2)
def test_conv_flatten_deriv(n4_hw12_c3_5x5): """ Test deriv of conv followed by flatten """ cf = ConvParams(**n4_hw12_c3_5x5) axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]]) axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length) for ax in axes_rsck]) axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]]) # broadcast input / filter axes input_var = ng.variable(cf.ax_i).named('input') input_val = np.ones(input_var.axes.lengths) filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter') filter_var = filter_rsck_prime filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck') filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck') filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk') # convolution output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o) output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output_npqk = ng.tensor_slice(output_nmpqk, out_slicing) output = ng.flatten_at(output_npqk, idx=1) # cost and grad cost = ng.sum(output, out_axes=()) filter_val = np.ones(filter_var.axes.lengths) with ExecutorFactory() as factory: conv_comp = factory.executor(output, filter_var, input_var) grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var) grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var) grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var) grad_input_sym_comp = factory.derivative(cost, input_var, filter_var) conv_val = conv_comp(filter_val, input_val) conv_val_num = np.empty_like(conv_val) conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1])) ng.testing.assert_allclose(conv_val, conv_val_num) grad_filter_num_val = grad_filter_num_comp(filter_val, input_val) grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val) ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val) grad_input_num_val = grad_input_num_comp(input_val, filter_val) grad_input_sym_val = grad_input_sym_comp(input_val, filter_val) ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def execute_convolution(image_height, image_width, filter_height, filter_width, channel=16, batch_size=32, filter_count=8, image_3rd_dim=1, filter_3rd_dim=1, padding=(0, 0, 0), stride=(1, 1, 1), dilation=1, np_comparison=False): pad_h, pad_w, pad_d = padding str_h, str_w, str_d = stride cf = ConvParams(C=channel, N=batch_size, K=filter_count, D=image_3rd_dim, H=image_height, W=image_width, T=filter_3rd_dim, R=filter_height, S=filter_width, pad_d=pad_d, pad_h=pad_h, pad_w=pad_w, str_d=str_d, str_h=str_h, str_w=str_w, dil_d=dilation, dil_h=dilation, dil_w=dilation) inputs = ng.placeholder(cf.ax_i) filters = ng.placeholder(cf.ax_f) rng = RandomTensorGenerator(0, np.float32) input_value = rng.uniform(-4, 4, cf.ax_i, dtype=int) filter_value = rng.uniform(-4, 4, cf.ax_f, dtype=int) error_value = rng.uniform(-0.5, 0.5, cf.ax_o) with executor( ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o), inputs, filters) as const_executor: out = const_executor(input_value, filter_value) if np_comparison: np_out, gradInp, gradF_np = \ reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value, error_value) return out, np_out return out
def test_wrong_input_shape_length(): """ test wrong input shape length """ cf = ConvParams() ax_i = cf.ax_i[:-1] inputs = ng.placeholder(ax_i) filters = ng.placeholder(cf.ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(cf.conv_params, inputs, filters, {}) assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\ .format(len(ax_i))
def test_first_axes_not_same(): """ test first axes are not the same """ cf = ConvParams() ax_i = cf.ax_i[1:2] + cf.ax_i[0:1] + cf.ax_i[2:] # D, C, H, W, N inputs = ng.placeholder(ax_i) filters = ng.placeholder(cf.ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(cf.conv_params, inputs, filters, {}) assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \ 'are not the same.'.format( inputs=inputs.axes[0], filters=filters.axes[0])
def test_convolution_backprop(n128_hw32_c3_2x2): """ test convolution backprop path """ cf = ConvParams(**n128_hw32_c3_2x2) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize input_value = rng.uniform(-1, 1, cf.ax_i) filter_value = rng.uniform(-1, 1, cf.ax_f) output = ng.sum(ng.convolution(cf.conv_params, inputs, filters, cf.ax_o), out_axes=()) with ExecutorFactory() as factory: dcdf_sym_fun = factory.derivative(output, filters, inputs) dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs) dcdf_sym_val = dcdf_sym_fun(filter_value, input_value) dcdf_num_val = dcdf_num_fun(filter_value, input_value) ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=0.01)
def test_deconv(transformer_factory, deconv_n4_hw4_c1_5x5): cf = ConvParams(**deconv_n4_hw4_c1_5x5) # randomly initialize input_value = rng.uniform(-0.5, 0.5, cf.ax_i) filter_value = rng.uniform(-0.5, 0.5, cf.ax_f) error_value = rng.uniform(-0.5, 0.5, cf.ax_o) inputs = ng.placeholder(cf.ax_i) filters = ng.placeholder(cf.ax_f) errors = ng.placeholder(cf.ax_o) output = ng.deconvolution(cf.conv_params, inputs, filters, axes=cf.ax_o) bprop_out = ng.deriv(output, inputs, errors) updat_out = ng.deriv(output, filters, errors) with executor([output, bprop_out, updat_out], inputs, filters, errors) as conv_executor: result_ng, gradI_ng, gradF_ng = conv_executor(input_value, filter_value, error_value) # Compute reference with NumPy result_np = reference_deconv_fprop(cf.conv_params, input_value, filter_value) gradI_np, gradF_np = reference_deconv_bprop(cf.conv_params, error_value, input_value, filter_value) # Compare fprop assert np.allclose(result_ng, result_np, rtol=0.1, atol=0) # Compare bprop assert np.allclose(gradI_ng, gradI_np, rtol=0.1, atol=0) # Compare update assert np.allclose(gradF_ng, gradF_np, rtol=0.1, atol=0)