def test_wrong_number_of_batch_axes_at_input(): """ test wrong number of batch axes at input """ padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) dilation = dict(dil_d=1, dil_h=1, dil_w=1) conv_params = padding.copy() conv_params.update(strides) conv_params.update(dilation) C = 3 D = 1 ax_C = ng.make_axis(name='N', length=C) ax_D = ng.make_axis(name='N', length=D) ax_i = ng.make_axes([ax_C, ax_D, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax_C, ax.T, ax.R, ax.S, ax.K]) inputs = ng.placeholder(axes=ax_i) filters = ng.placeholder(ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(conv_params, inputs, filters, {}) assert str(exinfo.value) == "Input must have one batch axis. Found {n_batch_axes} " \ "batch axes: {batch_axes} Found {n_sample_axes} sample axes: {sample_axes}.".format( n_batch_axes=len(inputs.axes.batch_axes()), batch_axes=inputs.axes.batch_axes(), n_sample_axes=len(inputs.axes.sample_axes()), sample_axes=inputs.axes.sample_axes())
def test_wrong_input_shape_length(): """ test wrong input shape length """ cf = ConvParams() ax_i = cf.ax_i[:-1] inputs = ng.placeholder(ax_i) filters = ng.placeholder(cf.ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(cf.conv_params, inputs, filters, {}) assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\ .format(len(ax_i))
def make_convolution_op(onnx_node, ng_inputs): # type: (NodeWrapper, List[NgraphNode]) -> NgraphNode """ Create an ngraph convolution Op based on an ONNX node. :param onnx_node: wrapped ONNX node for Conv of ConvTranspose op :param ng_inputs: ngraph TensorOp input tensors :return: ngraph Op for convolution or deconvolution """ if len(ng_inputs) == 3: x, weights, bias = ng_inputs elif len(ng_inputs) == 2: x, weights = ng_inputs bias = ng.constant(0, dtype=get_dtype(x.get_element_type())) else: raise ValueError( 'Conv node (%s): unexpected number of input values: %d.', onnx_node.name, len(ng_inputs)) groups = onnx_node.get_attribute_value('group', 1) if groups != 1: log.warning( 'Conv node (%s): `group` attribute value %d is not supported.', onnx_node.name, groups) strides = get_strides(onnx_node) dilation = get_dilations(onnx_node) padding_below, padding_above = get_pads(onnx_node) conv = ng.convolution(x, weights, strides, dilation, padding_below, padding_above) return conv + bias
def create_encoder(input_shape, levels = 4): import ngraph as ng # input input_node = ng.parameter(input_shape, np.float32, name="data") padding_begin = padding_end = [0, 0] strides = [1, 1] dilations = [1, 1] input_channels = [input_shape[1]] last_output = input_node # convolution layers for i in range(levels): input_c = input_channels[-1] output_c = input_c * 2 conv_w = np.random.uniform(0, 1, [output_c, input_c, 5, 5]).astype(np.float32) conv_node = ng.convolution(last_output, conv_w, strides, padding_begin, padding_end, dilations) input_channels.append(output_c) last_output = conv_node # deconvolution layers for i in range(levels): input_c = input_channels[-2] output_c = input_channels.pop(-1) deconv_w = np.random.uniform(0, 1, [output_c, input_c, 5, 5]).astype(np.float32) deconv_node = ng.convolution_backprop_data(last_output, deconv_w, strides) last_output = deconv_node # result last_output.set_friendly_name("out") result_node = ng.result(last_output) return ng.Function(result_node, [input_node], "Encoder")
def test_first_axes_not_same(): """ test first axes are not the same """ cf = ConvParams() ax_i = cf.ax_i[1:2] + cf.ax_i[0:1] + cf.ax_i[2:] # D, C, H, W, N inputs = ng.placeholder(ax_i) filters = ng.placeholder(cf.ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(cf.conv_params, inputs, filters, {}) assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \ 'are not the same.'.format( inputs=inputs.axes[0], filters=filters.axes[0])
def test_convolution_with_non_zero_padding(): element_type = Type.f32 image_shape = Shape([1, 1, 10, 10]) filter_shape = Shape([1, 1, 3, 3]) data = Parameter(element_type, image_shape) filters = Parameter(element_type, filter_shape) parameter_list = [data, filters] image_arr = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10) filter_arr = (np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3)) * -1 filter_arr[0][0][1][1] = 1 strides = [1, 1] dilations = [2, 2] pads_begin = [2, 1] pads_end = [1, 2] model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations) function = Function([model], parameter_list, "test") runtime = get_runtime() computation = runtime.computation(function, *parameter_list) result = computation(image_arr, filter_arr)[0] expected = convolution2d( image_arr[0][0], filter_arr[0][0], strides, dilations, pads_begin, pads_end ).reshape([1, 1, 9, 9]) assert np.allclose(result, expected)
def train_outputs(self, in_obj): cpm = self.convparams.copy() in_obj = ng.axes_with_role_order(in_obj, self.role_order) in_axes = in_obj.axes if self.f_axes is None: self.f_axes = ng.make_axes([in_axes[0]]) for nm, role in zip('TRSK', self.filter_roles[1:]): self.f_axes += ng.make_axis(roles=[role], length=cpm[nm]).named(nm) self.W = ng.variable(axes=self.f_axes, initial_value=self.init) if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(roles=a.roles).named(a.short_name) for a in in_axes if not a.is_batch ]) # set lengths out_shape = [ self.f_axes[-1].length, output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'], cpm['str_d'], False, cpm['dil_d']), output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'], cpm['str_h'], False, cpm['dil_h']), output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'], cpm['str_w'], False, cpm['dil_w']) ] self.o_axes.set_shape(out_shape) self.o_axes += in_axes.batch_axes() return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def test_conv(n64_hw32_c32_3x3): cf = ConvParams(**n64_hw32_c32_3x3) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize input_value = rng.uniform(-0.5, 0.5, cf.ax_i) filter_value = rng.uniform(-0.5, 0.5, cf.ax_f) error_value = rng.uniform(-0.5, 0.5, cf.ax_o) inputs = ng.placeholder(cf.ax_i) filters = ng.placeholder(cf.ax_f) errors = ng.placeholder(cf.ax_o) output = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o) bprop_out = bprop_conv(errors, inputs, filters, output) updat_out = update_conv(errors, inputs, filters, output) with executor([output, bprop_out, updat_out], inputs, filters, errors) as conv_executor: result_ng, gradI_ng, gradF_ng = conv_executor(input_value, filter_value, error_value) # Compute reference with NumPy result_np, gradI_np, gradF_np = reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value, error_value) # Compare fprop assert np.allclose(result_ng, result_np, rtol=0, atol=0.5) # Compare bprop assert np.allclose(gradI_ng, gradI_np, rtol=0, atol=0.5) # Compare update assert np.allclose(gradF_ng, gradF_np, rtol=0, atol=2)
def test_convolution_simple(): element_type = Type.f32 image_shape = Shape([1, 1, 16, 16]) filter_shape = Shape([1, 1, 3, 3]) data = Parameter(element_type, image_shape) filters = Parameter(element_type, filter_shape) parameter_list = [data, filters] image_arr = np.arange(-128, 128, 1, dtype=np.float32).reshape(1, 1, 16, 16) filter_arr = np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3) filter_arr[0][0][0][0] = -1 filter_arr[0][0][1][1] = -1 filter_arr[0][0][2][2] = -1 filter_arr[0][0][0][2] = -1 filter_arr[0][0][2][0] = -1 strides = [1, 1] pads_begin = [0, 0] pads_end = [0, 0] dilations = [1, 1] model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations) function = Function([model], parameter_list, "test") runtime = get_runtime() computation = runtime.computation(function, *parameter_list) result = computation(image_arr, filter_arr)[0] expected = convolution2d(image_arr[0][0], filter_arr[0][0]).reshape(1, 1, 14, 14) assert np.allclose(result, expected)
def test_conv_flatten_deriv(n4_hw12_c3_5x5): """ Test deriv of conv followed by flatten """ cf = ConvParams(**n4_hw12_c3_5x5) axes_rsck = ng.make_axes([cf.ax_f[2], cf.ax_f[3], cf.ax_f[0], cf.ax_f[-1]]) axes_rsck_prime = ng.make_axes([ng.make_axis(name=ax.name + 'p', length=ax.length) for ax in axes_rsck]) axes_nmpqk = ng.make_axes([cf.ax_o[-1], cf.ax_o[1], cf.ax_o[2], cf.ax_o[3], cf.ax_o[0]]) # broadcast input / filter axes input_var = ng.variable(cf.ax_i).named('input') input_val = np.ones(input_var.axes.lengths) filter_rsck_prime = ng.variable(axes_rsck_prime).named('filter') filter_var = filter_rsck_prime filter_rsck = ng.cast_axes(filter_rsck_prime, axes_rsck).named('frsck') filter_trsck = ng.expand_dims(filter_rsck, cf.ax_f[1], 0).named('ftrsck') filter_ctrsk = ng.axes_with_order(filter_trsck, axes=cf.ax_f).named('ctrsk') # convolution output_kmpqn = ng.convolution(cf.conv_params, input_var, filter_ctrsk, axes=cf.ax_o) output_nmpqk = ng.axes_with_order(output_kmpqn, axes=axes_nmpqk) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output_npqk = ng.tensor_slice(output_nmpqk, out_slicing) output = ng.flatten_at(output_npqk, idx=1) # cost and grad cost = ng.sum(output, out_axes=()) filter_val = np.ones(filter_var.axes.lengths) with ExecutorFactory() as factory: conv_comp = factory.executor(output, filter_var, input_var) grad_filter_num_comp = factory.numeric_derivative(cost, filter_var, 1.0, input_var) grad_filter_sym_comp = factory.derivative(cost, filter_var, input_var) grad_input_num_comp = factory.numeric_derivative(cost, input_var, 1.0, filter_var) grad_input_sym_comp = factory.derivative(cost, input_var, filter_var) conv_val = conv_comp(filter_val, input_val) conv_val_num = np.empty_like(conv_val) conv_val_num.fill(np.prod(cf.ax_f.lengths[:-1])) ng.testing.assert_allclose(conv_val, conv_val_num) grad_filter_num_val = grad_filter_num_comp(filter_val, input_val) grad_filter_sym_val = grad_filter_sym_comp(filter_val, input_val) ng.testing.assert_allclose(grad_filter_num_val, grad_filter_sym_val) grad_input_num_val = grad_input_num_comp(input_val, filter_val) grad_input_sym_val = grad_input_sym_comp(input_val, filter_val) ng.testing.assert_allclose(grad_input_num_val, grad_input_sym_val)
def test_wrong_input_shape_length(): """ test wrong input shape length """ padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) conv_params = padding.copy() conv_params.update(strides) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) inputs = ng.placeholder(ax_i) filters = ng.placeholder(ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(conv_params, inputs, filters, {}) assert str(exinfo.value) == 'convolution input shape must be length 5, found {}'\ .format(len(ax_i))
def test_first_axes_not_same(): """ test first axes are not the same """ padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) conv_params = padding.copy() conv_params.update(strides) ax_i = ng.make_axes([ax.D, ax.C, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) inputs = ng.placeholder(ax_i) filters = ng.placeholder(ax_f) with pytest.raises(ValueError) as exinfo: ng.convolution(conv_params, inputs, filters, {}) assert str(exinfo.value) == 'the first axis in input {inputs} and filter {filters} ' \ 'are not the same.'.format( inputs=inputs.axes[0], filters=filters.axes[0])
def test_convolution_backprop(transformer_factory): """ test convolution backprop path """ N = 128 C, K = 3, 2 D, T = 1, 1 H = W = 32 R = S = 2 padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) dilation = dict(dil_d=1, dil_h=1, dil_w=1) conv_params = padding.copy() conv_params.update(strides) conv_params.update(dilation) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) ax_i.set_shape((C, D, H, W, N)) ax_f.set_shape((C, T, R, S, K)) ax_o = ng.make_axes([ ng.make_axis(roles=[ar.features_input]).named('C'), ng.make_axis(roles=[ar.features_0]).named('D'), ng.make_axis(roles=[ar.features_1]).named('H'), ng.make_axis(roles=[ar.features_2]).named('W'), ax.N ]) ax_o[:-1].set_shape((K, output_dim(D, T, padding['pad_d'], strides['str_d']), output_dim(H, R, padding['pad_h'], strides['str_h']), output_dim(W, S, padding['pad_w'], strides['str_w']))) inputs = ng.placeholder(axes=ax_i) filters = ng.placeholder(axes=ax_f) # randomly initialize input_value = rng.uniform(-1, 1, ax_i) filter_value = rng.uniform(-1, 1, ax_f) assert input_value.shape == ax_i.lengths assert filter_value.shape == ax_f.lengths output = ng.sum(ng.convolution(conv_params, inputs, filters, ax_o), out_axes=()) with ExecutorFactory() as factory: dcdf_sym_fun = factory.derivative(output, filters, inputs) dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs) dcdf_sym_val = dcdf_sym_fun(filter_value, input_value) dcdf_num_val = dcdf_num_fun(filter_value, input_value) ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=1)
def execute_convolution(image_height, image_width, filter_height, filter_width, channel=16, batch_size=32, filter_count=8, image_3rd_dim=1, filter_3rd_dim=1, padding=(0, 0, 0), stride=(1, 1, 1), dilation=1, np_comparison=False): pad_h, pad_w, pad_d = padding str_h, str_w, str_d = stride cf = ConvParams(C=channel, N=batch_size, K=filter_count, D=image_3rd_dim, H=image_height, W=image_width, T=filter_3rd_dim, R=filter_height, S=filter_width, pad_d=pad_d, pad_h=pad_h, pad_w=pad_w, str_d=str_d, str_h=str_h, str_w=str_w, dil_d=dilation, dil_h=dilation, dil_w=dilation) inputs = ng.placeholder(cf.ax_i) filters = ng.placeholder(cf.ax_f) rng = RandomTensorGenerator(0, np.float32) input_value = rng.uniform(-4, 4, cf.ax_i, dtype=int) filter_value = rng.uniform(-4, 4, cf.ax_f, dtype=int) error_value = rng.uniform(-0.5, 0.5, cf.ax_o) with executor( ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o), inputs, filters) as const_executor: out = const_executor(input_value, filter_value) if np_comparison: np_out, gradInp, gradF_np = \ reference_conv(cf.dimI, cf.dimF, cf.dimO, cf.conv_params, input_value, filter_value, error_value) return out, np_out return out
def test_conv_flatten_deriv(transformer_factory): """ Test deriv of conv followed by flatten """ # set shape C, D, H, W, N = (3, 1, 28, 28, 8) C, T, R, S, K = (3, 1, 5, 5, 32) # i, f, o axes ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) ax_o = ng.make_axes([ ng.make_axis(32, roles=[ar.Channel]), ng.make_axis(1, roles=[ar.Depth]), ng.make_axis(24, roles=[ar.Height]), ng.make_axis(24, roles=[ar.Width]), ax.N ]) ax_i.set_shape((C, D, H, W, N)) ax_f.set_shape((C, T, R, S, K)) params = dict(pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1) axes_rsck = ng.make_axes([ax.R, ax.S, ax.C, ax.K]) axes_rsck_prime = ng.make_axes( [ng.make_axis(l) for l in axes_rsck.lengths]) # broadcast input / filter axes image = ng.constant(np.ones(ax_i.lengths), ax_i) filter = ng.variable(axes_rsck_prime, initial_value=np.ones((R, S, C, K))) filter_casted = ng.cast_axes(filter, axes_rsck) filter_casted = ng.expand_dims(filter_casted, ax.T, 0) filter_casted = ng.axes_with_order(filter_casted, axes=ax_f) # convolution output = ng.convolution(params, image, filter_casted, axes=ax_o) oC, oD, oH, oW, oN = output.axes output = ng.axes_with_order(output, axes=ng.make_axes([oN, oD, oH, oW, oC])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] conv = ng.Slice(output, out_slicing) flatten = ng.flatten_at(conv, idx=1) # cost and grad cost = ng.sum(flatten, reduction_axes=flatten.axes) grad = ng.deriv(cost, filter) # compute conv_grad_comp = executor([conv, grad]) conv_val, grad_val = conv_grad_comp() assert np.allclose(conv_val, np.zeros_like(conv_val) + 75.) assert np.allclose(grad_val, np.zeros_like(grad_val) + 4608.)
def _convolution_op(self, cntk_op, inputs): """ Computes the convolution of a tensor with operand. CNTK Ngraph in ((C, H, W) N)) (C, D, H, W, N) filter (O, C, M1, M2) (C, T, M1, M2, O) out ((O, H, W) N) (O, M, H, W, N) Arguments: cntk_op: CNTK operation to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ filters, inputs = inputs inputs = self._expand_input_axes(inputs) C, D, H, W, N = inputs.axes filters = self._expand_filters_axes(filters, C) _, T, M1, M2, O = filters.axes M, oH, oW = self._make_out_axes(cntk_op.shape) out_axes = [O, M, oH, oW, N] strides = self._make_strides(cntk_op.attributes['strides']) pad = self._make_padding('conv', cntk_op.attributes['autoPadding'], (D.length, H.length, W.length, C.length), (T.length, M1.length, M2.length, C.length), (M.length, oH.length, oW.length, O.length), strides) params = dict(pad_d=pad[0], pad_h=pad[1], pad_w=pad[2], pad_c=pad[3], str_d=strides[0], str_h=strides[1], str_w=strides[2], str_c=strides[3], dil_d=1, dil_h=1, dil_w=1) conv = ng.convolution(params, inputs, filters, out_axes) return remove_ones_axes([conv])[0]
def test_convolution_simple(): element_type = Type.f32 image_shape = Shape([1, 1, 16, 16]) filter_shape = Shape([1, 1, 3, 3]) data = Parameter(element_type, image_shape) filters = Parameter(element_type, filter_shape) parameter_list = [data, filters] image_arr = np.arange(-128, 128, 1, dtype=np.float32).reshape(1, 1, 16, 16) filter_arr = np.ones(9, dtype=np.float32).reshape(1, 1, 3, 3) filter_arr[0][0][0][0] = -1 filter_arr[0][0][1][1] = -1 filter_arr[0][0][2][2] = -1 filter_arr[0][0][0][2] = -1 filter_arr[0][0][2][0] = -1 result_arr = np.zeros(196, dtype=np.float32).reshape(1, 1, 14, 14) strides = [1, 1] pads_begin = [0, 0] pads_end = [0, 0] dilations = [1, 1] model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations) function = Function([model], parameter_list, "test") backend = Backend.create(test.BACKEND_NAME) a = backend.create_tensor(element_type, image_shape) b = backend.create_tensor(element_type, filter_shape) a.write(util.numpy_to_c(image_arr), 16 * 16 * 4) b.write(util.numpy_to_c(filter_arr), 3 * 3 * 4) result = backend.create_tensor(element_type, Shape([1, 1, 14, 14])) result.write(util.numpy_to_c(result_arr), 14 * 14 * 4) handle = backend.compile(function) handle.call([result], [a, b]) result.read(util.numpy_to_c(result_arr), 14 * 14 * 4) result_arr_ref = convolution2d(image_arr[0][0], filter_arr[0][0]).reshape(1, 1, 14, 14) assert np.allclose(result_arr, result_arr_ref)
def test_convolution_backprop(n128_hw32_c3_2x2): """ test convolution backprop path """ cf = ConvParams(**n128_hw32_c3_2x2) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize input_value = rng.uniform(-1, 1, cf.ax_i) filter_value = rng.uniform(-1, 1, cf.ax_f) output = ng.sum(ng.convolution(cf.conv_params, inputs, filters, cf.ax_o), out_axes=()) with ExecutorFactory() as factory: dcdf_sym_fun = factory.derivative(output, filters, inputs) dcdf_num_fun = factory.numeric_derivative(output, filters, .01, inputs) dcdf_sym_val = dcdf_sym_fun(filter_value, input_value) dcdf_num_val = dcdf_num_fun(filter_value, input_value) ng.testing.assert_allclose(dcdf_sym_val, dcdf_num_val, rtol=0.01)
def test_convolution_with_padding(): element_type = Type.f32 image_shape = Shape([1, 1, 10, 10]) filter_shape = Shape([1, 1, 3, 3]) data = Parameter(element_type, image_shape) filters = Parameter(element_type, filter_shape) parameter_list = [data, filters] image_arr = np.arange(100, dtype=np.float32).reshape(1, 1, 10, 10) filter_arr = np.zeros(9, dtype=np.float32).reshape(1, 1, 3, 3) filter_arr[0][0][1][1] = 1 strides = [1, 1] dilations = [2, 2] pads_begin = [0, 0] pads_end = [0, 0] model = ng.convolution(data, filters, strides, pads_begin, pads_end, dilations) function = Function([model], parameter_list, "test") backend = Backend.create(test.BACKEND_NAME) a = backend.create_tensor(element_type, image_shape) b = backend.create_tensor(element_type, filter_shape) a.write(util.numpy_to_c(image_arr), 10 * 10 * 4) b.write(util.numpy_to_c(filter_arr), 3 * 3 * 4) result_arr = np.zeros(36, dtype=np.float32).reshape(1, 1, 6, 6) result = backend.create_tensor(element_type, Shape([1, 1, 6, 6])) result.write(util.numpy_to_c(result_arr), 6 * 6 * 4) handle = backend.compile(function) handle.call([result], [a, b]) result.read(util.numpy_to_c(result_arr), 6 * 6 * 4) result_arr_ref = convolution2d(image_arr[0][0], filter_arr[0][0], strides, dilations, pads_begin, pads_end).reshape(1, 1, 6, 6) assert np.allclose(result_arr, result_arr_ref)
def train_outputs(self, in_obj): cpm = self.convparams.copy() in_axes = in_obj.axes if self.f_axes is None: self.f_axes = in_axes.role_axes(ar.Channel) for _ax in (ax.T, ax.R, ax.S, ax.K): self.f_axes += ng.make_axis(roles=_ax.roles).named(_ax.short_name) self.f_axes[1:].set_shape(itemgetter(*'TRSK')(cpm)) self.W = ng.variable(axes=self.f_axes, initial_value=self.init(self.f_axes.lengths)) # TODO: clean this up if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(self.f_axes[4].length, roles=[ar.Channel]).named('C'), ng.spatial_axis(in_axes, self.f_axes, cpm['pad_d'], cpm['str_d'], role=ar.Depth), ng.spatial_axis(in_axes, self.f_axes, cpm['pad_h'], cpm['str_h'], role=ar.Height), ng.spatial_axis(in_axes, self.f_axes, cpm['pad_w'], cpm['str_w'], role=ar.Width), ax.N ]) return ng.convolution(cpm, in_obj, self.W, axes=self.o_axes)
def __call__(self, in_obj): cpm = self.convparams.copy() in_obj = reorder_spatial_axes(in_obj) in_axes = in_obj.axes if self.f_axes is None: self.f_axes = ng.make_axes([in_axes[0]]) for nm in 'TRSK': self.f_axes |= ng.make_axis(length=cpm[nm], name=nm) # mark 'K' as a shadow axis for the initializers. self.axes_map = shadow_axes_map(self.f_axes.find_by_name('K')) self.f_axes = ng.make_axes([ axis if axis.name != 'K' else list(self.axes_map.keys())[0] for axis in self.f_axes ]) self.W = ng.variable(axes=self.f_axes, initial_value=self.init, scope=self.scope).named('convwt') if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(name=a.name) for a in in_axes if not a.is_batch ]) # set lengths out_shape = [ self.f_axes[-1].length, output_dim(in_axes[1].length, cpm['T'], cpm['pad_d'], cpm['str_d'], False, cpm['dil_d']), output_dim(in_axes[2].length, cpm['R'], cpm['pad_h'], cpm['str_h'], False, cpm['dil_h']), output_dim(in_axes[3].length, cpm['S'], cpm['pad_w'], cpm['str_w'], False, cpm['dil_w']) ] self.o_axes.set_shape(out_shape) self.o_axes |= in_axes.batch_axis() return ng.map_roles(ng.convolution(cpm, in_obj, self.W, axes=self.o_axes), self.axes_map)
def Conv(self, c2_op, inputs): """ Computes a 2-D convolution given 4D input and filter tensors. Arguments: c2_op: NodeDef object, the caffe2 node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the caffe2 node. Inputs to c2_op: input, wegiths, filter Supports caffe2's layout NHWC and NCHW as well. """ X, W, bias = inputs order = [val.s for val in c2_op.arg if val.name == "order"] if 1 != len(order): raise ValueError("Multiple order values in convolution") order = order[0] if order not in ("NHWC", "NCHW"): raise NotImplementedError("Unsupported order in convolution: {}", order) # set input axes shape ax_N = ng.make_axis(name='N') ax_C = ng.make_axis() ax_D = ng.make_axis(length=1) ax_H = ng.make_axis() ax_W = ng.make_axis() # set kernel axes shape ax_kernel_D = ng.make_axis(length=1) ax_kernel_H = ng.make_axis() ax_kernel_W = ng.make_axis() ax_kernel_ofm = ng.make_axis() # create placeholders for output axes oC = ng.make_axis(name='C') oD = ng.make_axis(name='D', length=1) oH = ng.make_axis(name='H') oW = ng.make_axis(name='W') axes_order = { 'NCHW': { 'X': [ax_N, ax_C, ax_H, ax_W], 'W': [ax_kernel_ofm, ax_C, ax_kernel_H, ax_kernel_W] }, 'NHWC': { 'X': [ax_N, ax_H, ax_W, ax_C], 'W': [ax_kernel_ofm, ax_kernel_H, ax_kernel_W, ax_C] }, } ng.make_axes(axes_order[order]['X']).set_shape(X.axes.lengths) ng.make_axes(axes_order[order]['W']).set_shape(W.axes.lengths) if 1 != len(bias.axes): raise ValueError("Bias's must be 1D.") if ax_kernel_ofm.length != bias.axes.lengths[0]: raise ValueError( "Bias's length must equal to number of output feature maps.") # strides params stride_size = [int(val.i) for val in c2_op.arg if val.name == "stride"] if len(stride_size) != 1: raise ValueError("Stride size must be scalar value") str_h = str_w = stride_size[0] # padding params pad_t, pad_b, pad_l, pad_r = \ _c2_padding(c2_op, in_NHWC=[ax_N.length, ax_H.length, ax_W.length, ax_C.length], kernel_HWIO=[ax_kernel_H.length, ax_kernel_W.length, ax_C.length, ax_kernel_ofm.length], stride_NHWC=[1, str_h, str_w, 1]) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # conv params params = dict(pad_d=0, pad_h=pad_t, pad_w=pad_l, str_d=1, str_h=str_h, str_w=str_w, dil_d=1, dil_h=1, dil_w=1) # input, weight, output axes internal_ax_dict = { 'X': ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]), 'W': ng.make_axes( [ax_C, ax_kernel_D, ax_kernel_H, ax_kernel_W, ax_kernel_ofm]) } oC.length = ax_kernel_ofm.length oH.length = output_dim(ax_H.length, ax_kernel_H.length, params['pad_h'], params['str_h']) oW.length = output_dim(ax_W.length, ax_kernel_W.length, params['pad_w'], params['str_w']) internal_ax_dict['Y'] = ng.make_axes([oC, oD, oH, oW, ax_N]) # broadcast input / filter axes # flow for NHWC order: | flow for NCHW order: # input: | input: # expand dims: NHWC -> NDHWC | expand dims: NCHW -> NDCHW # reorder: NDHWC -> CDHWN | reorder: NDCHW -> CDHWN # weights: | weights: # expand dims: (ofm)HWC -> D(ofm)HWC | expand dims: (ofm)CHWC -> D(ofm)CHW # reorder: D(ofm)HWC -> CDHW(ofm) | reorder: D(ofm)CHW -> CDHW(ofm) X = ng.cast_axes(X, ng.make_axes(axes_order[order]['X'])) X = ng.expand_dims(X, ax_D, 1) X = ng.axes_with_order(X, axes=internal_ax_dict['X']) W = ng.cast_axes(W, ng.make_axes(axes_order[order]['W'])) W = ng.expand_dims(W, ax_kernel_D, 0) W = ng.axes_with_order(W, axes=internal_ax_dict['W']) # convolution Y = ng.convolution(params, X, W, axes=internal_ax_dict['Y']) # cast back to proper format Y = ng.broadcast(Y, ng.make_axes([ax_N, oD, oH, oW, oC])) if "NHWC" == order \ else ng.broadcast(Y, ng.make_axes([ax_N, oD, oC, oH, oW])) # NCHW # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] Y = ng.tensor_slice(Y, out_slicing) def _conv_bias_add(c2_op, inputs): X, bias = inputs bias = ng.cast_axes(bias, axes=ng.make_axes( [X.axes[1 if 'NCHW' == order else 3]])) Y = ng.Add(X, bias) return Y return _conv_bias_add(c2_op, [Y, bias])
def create_ngraph_function(args) -> Function: weights = np.fromfile(args.model, dtype=np.float32) weights_offset = 0 padding_begin = [0, 0] padding_end = [0, 0] # input input_shape = [64, 1, 28, 28] param_node = ngraph.parameter(input_shape, np.float32, 'Parameter') # convolution 1 conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5]) conv_1_kernel = ngraph.constant( weights[0:conv_1_kernel_length].reshape(conv_1_kernel_shape)) weights_offset += conv_1_kernel_length conv_1_node = ngraph.convolution(param_node, conv_1_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 1 add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1]) add_1_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_1_kernel_length].reshape(add_1_kernel_shape)) weights_offset += add_1_kernel_length add_1_node = ngraph.add(conv_1_node, add_1_kernel) # maxpool 1 maxpool_1_node = ngraph.max_pool(add_1_node, [2, 2], padding_begin, padding_end, [2, 2], 'ceil', None) # convolution 2 conv_2_kernel_shape, conv_2_kernel_length = shape_and_length( [50, 20, 5, 5]) conv_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + conv_2_kernel_length].reshape(conv_2_kernel_shape)) weights_offset += conv_2_kernel_length conv_2_node = ngraph.convolution(maxpool_1_node, conv_2_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 2 add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1]) add_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_2_kernel_length].reshape(add_2_kernel_shape)) weights_offset += add_2_kernel_length add_2_node = ngraph.add(conv_2_node, add_2_kernel) # maxpool 2 maxpool_2_node = ngraph.max_pool(add_2_node, [2, 2], padding_begin, padding_end, [2, 2], 'ceil', None) # reshape 1 reshape_1_dims, reshape_1_length = shape_and_length([2]) # workaround to get int64 weights from float32 ndarray w/o unnecessary copying dtype_weights = np.frombuffer(weights[weights_offset:weights_offset + 2 * reshape_1_length], dtype=np.int64) reshape_1_kernel = ngraph.constant(dtype_weights) weights_offset += 2 * reshape_1_length reshape_1_node = ngraph.reshape(maxpool_2_node, reshape_1_kernel, True) # matmul 1 matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length( [500, 800]) matmul_1_kernel = ngraph.constant( weights[weights_offset:weights_offset + matmul_1_kernel_length].reshape(matmul_1_kernel_shape)) weights_offset += matmul_1_kernel_length matmul_1_node = ngraph.matmul(reshape_1_node, matmul_1_kernel, False, True) # add 3 add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500]) add_3_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_3_kernel_length].reshape(add_3_kernel_shape)) weights_offset += add_3_kernel_length add_3_node = ngraph.add(matmul_1_node, add_3_kernel) # ReLU relu_node = ngraph.relu(add_3_node) # reshape 2 reshape_2_kernel = ngraph.constant(dtype_weights) reshape_2_node = ngraph.reshape(relu_node, reshape_2_kernel, True) # matmul 2 matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500]) matmul_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + matmul_2_kernel_length].reshape(matmul_2_kernel_shape)) weights_offset += matmul_2_kernel_length matmul_2_node = ngraph.matmul(reshape_2_node, matmul_2_kernel, False, True) # add 4 add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10]) add_4_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_4_kernel_length].reshape(add_4_kernel_shape)) weights_offset += add_4_kernel_length add_4_node = ngraph.add(matmul_2_node, add_4_kernel) # softmax softmax_axis = 1 softmax_node = ngraph.softmax(add_4_node, softmax_axis) # result result_node = ngraph.result(softmax_node) # nGraph function function = Function(result_node, [param_node], 'lenet') return function
def test_convolution(transformer_factory): """ test convolution forward path """ N = 128 C, K = 3, 8 D, T = 1, 1 H = W = 32 R = S = 2 padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) conv_params = padding.copy() conv_params.update(strides) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) ax_i.set_shape((C, D, H, W, N)) ax_f.set_shape((C, T, R, S, K)) ax_o = ng.make_axes([ ng.make_axis(ax_f.role_axes(ar.Channelout)[0].length, name='C', roles=[ar.Channel]), spatial_axis(ax_i, ax_f, padding['pad_d'], strides['str_d'], role=ar.Depth), spatial_axis(ax_i, ax_f, padding['pad_h'], strides['str_h'], role=ar.Height), spatial_axis(ax_i, ax_f, padding['pad_w'], strides['str_w'], role=ar.Width), ax.N ]) inputs = ng.placeholder(axes=ax_i) filters = ng.placeholder(axes=ax_f) # randomly initialize input_value = rng.uniform(-1, 1, ax_i) filter_value = rng.uniform(-1, 1, ax_f) assert input_value.shape == ax_i.lengths assert filter_value.shape == ax_f.lengths inputs = ng.placeholder(ax_i) filters = ng.placeholder(ax_f) output = ng.convolution(conv_params, inputs, filters, axes=ax_o) targets = ng.placeholder(axes=output.axes) costs = ng.cross_entropy_binary(ng.sigmoid(output), targets) error = ng.sum(costs, out_axes=()) / ng.batch_size(costs) d_inputs = ng.deriv(error, inputs) d_filters = ng.deriv(error, filters) targets_value = rng.uniform(.1, 0.9, output.axes) conv_executor = executor([output, error, d_inputs, d_filters], inputs, filters, targets) result_ng, err_ng, gradI_ng, gradF_ng = conv_executor( input_value, filter_value, targets_value) # Now compute reference values via NEON NervanaObject.be.bsz = N neon_layer = Convolution(fshape=(R, S, K), padding=padding, strides=strides) inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N)) neon_layer.W = neon_layer.be.array(filter_value.reshape(C * R * S * T, K)) neon_layer.dW = neon_layer.be.empty_like(neon_layer.W) neon_layer.configure((C, H, W)) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.set_deltas(DummyDeltaBuffers()) result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths) act_result_ne = 1. / (1.0 + np.exp(-result_ne)) err = neon_layer.be.array( (act_result_ne - targets_value).reshape(-1, N) / float(N)) gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths) gradF_ne = neon_layer.dW.get().reshape(ax_f.lengths) # Compare fprop np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6) # Compare bprop np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6) # Compare update np.testing.assert_allclose(gradF_ng, gradF_ne, rtol=0, atol=1e-4)
def create_ngraph_function(args: argparse.Namespace) -> ngraph.impl.Function: """Create a network on the fly from the source code using ngraph""" def shape_and_length(shape: list) -> typing.Tuple[list, int]: length = reduce(lambda x, y: x * y, shape) return shape, length weights = np.fromfile(args.model, dtype=np.float32) weights_offset = 0 padding_begin = padding_end = [0, 0] # input input_shape = [64, 1, 28, 28] param_node = ngraph.parameter(input_shape, np.float32, 'Parameter') # convolution 1 conv_1_kernel_shape, conv_1_kernel_length = shape_and_length([20, 1, 5, 5]) conv_1_kernel = ngraph.constant( weights[0:conv_1_kernel_length].reshape(conv_1_kernel_shape)) weights_offset += conv_1_kernel_length conv_1_node = ngraph.convolution(param_node, conv_1_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 1 add_1_kernel_shape, add_1_kernel_length = shape_and_length([1, 20, 1, 1]) add_1_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_1_kernel_length].reshape(add_1_kernel_shape), ) weights_offset += add_1_kernel_length add_1_node = ngraph.add(conv_1_node, add_1_kernel) # maxpool 1 maxpool_1_node = ngraph.max_pool(add_1_node, [2, 2], padding_begin, padding_end, [2, 2], 'ceil', None) # convolution 2 conv_2_kernel_shape, conv_2_kernel_length = shape_and_length( [50, 20, 5, 5]) conv_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + conv_2_kernel_length].reshape(conv_2_kernel_shape), ) weights_offset += conv_2_kernel_length conv_2_node = ngraph.convolution(maxpool_1_node, conv_2_kernel, [1, 1], padding_begin, padding_end, [1, 1]) # add 2 add_2_kernel_shape, add_2_kernel_length = shape_and_length([1, 50, 1, 1]) add_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_2_kernel_length].reshape(add_2_kernel_shape), ) weights_offset += add_2_kernel_length add_2_node = ngraph.add(conv_2_node, add_2_kernel) # maxpool 2 maxpool_2_node = ngraph.max_pool(add_2_node, [2, 2], padding_begin, padding_end, [2, 2], 'ceil', None) # reshape 1 reshape_1_dims, reshape_1_length = shape_and_length([2]) # workaround to get int64 weights from float32 ndarray w/o unnecessary copying dtype_weights = np.frombuffer( weights[weights_offset:weights_offset + 2 * reshape_1_length], dtype=np.int64, ) reshape_1_kernel = ngraph.constant(dtype_weights) weights_offset += 2 * reshape_1_length reshape_1_node = ngraph.reshape(maxpool_2_node, reshape_1_kernel, True) # matmul 1 matmul_1_kernel_shape, matmul_1_kernel_length = shape_and_length( [500, 800]) matmul_1_kernel = ngraph.constant( weights[weights_offset:weights_offset + matmul_1_kernel_length].reshape(matmul_1_kernel_shape), ) weights_offset += matmul_1_kernel_length matmul_1_node = ngraph.matmul(reshape_1_node, matmul_1_kernel, False, True) # add 3 add_3_kernel_shape, add_3_kernel_length = shape_and_length([1, 500]) add_3_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_3_kernel_length].reshape(add_3_kernel_shape), ) weights_offset += add_3_kernel_length add_3_node = ngraph.add(matmul_1_node, add_3_kernel) # ReLU relu_node = ngraph.relu(add_3_node) # reshape 2 reshape_2_kernel = ngraph.constant(dtype_weights) reshape_2_node = ngraph.reshape(relu_node, reshape_2_kernel, True) # matmul 2 matmul_2_kernel_shape, matmul_2_kernel_length = shape_and_length([10, 500]) matmul_2_kernel = ngraph.constant( weights[weights_offset:weights_offset + matmul_2_kernel_length].reshape(matmul_2_kernel_shape), ) weights_offset += matmul_2_kernel_length matmul_2_node = ngraph.matmul(reshape_2_node, matmul_2_kernel, False, True) # add 4 add_4_kernel_shape, add_4_kernel_length = shape_and_length([1, 10]) add_4_kernel = ngraph.constant( weights[weights_offset:weights_offset + add_4_kernel_length].reshape(add_4_kernel_shape), ) weights_offset += add_4_kernel_length add_4_node = ngraph.add(matmul_2_node, add_4_kernel) # softmax softmax_axis = 1 softmax_node = ngraph.softmax(add_4_node, softmax_axis) # result result_node = ngraph.result(softmax_node) return ngraph.impl.Function(result_node, [param_node], 'lenet')
def test_default_arguments_convolution_2d(): manager_name = pytest.config.getoption('backend', default='CPU') runtime = ng.runtime(manager_name=manager_name) # input_x should have shape N(batch) x C x H x W input_x = ng.constant( np.array([[0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.]], dtype=np.float32).reshape(1, 1, 9, 9)) # filter weights should have shape M x C x kH x kW input_filter = ng.constant( np.array([[1., 0., -1.], [2., 0., -2.], [1., 0., -1.]], dtype=np.float32).reshape(1, 1, 3, 3)) # convolution with padding=1 should produce 9 x 9 output: model = runtime.computation( ng.convolution(input_x, input_filter, padding_above=[1, 1], padding_below=[1, 1])) result = model() assert np.array_equal( result, np.array([[[[0., -15., -15., 15., 15., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -15., -15., 15., 15., 0., 0., 0., 0.]]]], dtype=np.float32)) # convolution with padding=0 should produce 7 x 7 output: model = runtime.computation(ng.convolution(input_x, input_filter)) result = model() assert np.array_equal( result, np.array([[[[-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0]]]], dtype=np.float32)) # convolution with strides=2 should produce 4 x 4 output: model = runtime.computation( ng.convolution(input_x, input_filter, strides=[2, 2])) result = model() assert np.array_equal( result, np.array([[[[-20., 20., 0., 0.], [-20., 20., 0., 0.], [-20., 20., 0., 0.], [-20., 20., 0., 0.]]]], dtype=np.float32)) # convolution with dilation=2 should produce 5 x 5 output: model = runtime.computation( ng.convolution(input_x, input_filter, dilation=(2, 2))) result = model() assert np.array_equal( result, np.array([[[[0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0]]]], dtype=np.float32))
def test_recvop_tensorupdate(): """ The tensor (RecvOp_#_#) associated with the following conv op has two views: 1) Non-flat view (e.g. RecvOp_#_#_1_1_1_1_4.shape=(1,1,1,1,4)) 2) Flat view (e.g. RecvOp_#_#_1_4.shape = (1,4)) This test ensures that inside RecvOp code generation, the generated code should make sure both views get updated (e.g. by using update_RecvOp_#_# API) In this test, ng.dot operation tends to use the flat view (i.e. RecvOp_#_#_1_4) And previously RecvOp with RecvOp_#_#_1_1_1_1_4 = recv_from_send(send_id) failed to update both two views (i.e. flat and non-flat view of the same buffer/tensor) """ class ConvParams(object): def __init__(self, C=1, N=1, K=1, D=1, H=1, W=1, T=1, R=1, S=1, pad_d=0, pad_h=0, pad_w=0, str_d=1, str_h=1, str_w=1): from ngraph.frontends.common.utils import conv_output_dim M = conv_output_dim(D, T, pad_d, str_d) P = conv_output_dim(H, R, pad_h, str_h) Q = conv_output_dim(W, S, pad_w, str_w) self.dimO = (K, M, P, Q, N) self.dimI = (C, D, H, W, N) self.dimF = (C, T, R, S, K) self.conv_params = dict(pad_d=pad_d, pad_h=pad_h, pad_w=pad_w, str_d=str_d, str_h=str_h, str_w=str_w, dil_d=1, dil_h=1, dil_w=1) self.batch_axis = ng.make_axis(name='N', length=N) self.ax_i = ng.make_axes([ ng.make_axis(name='C', length=C), ng.make_axis(name='D', length=D), ng.make_axis(name='H', length=H), ng.make_axis(name='W', length=W), self.batch_axis ]) self.ax_f = ng.make_axes([ ng.make_axis(name='C', length=C), ng.make_axis(name='D', length=T), ng.make_axis(name='H', length=R), ng.make_axis(name='W', length=S), ng.make_axis(name='K', length=K), ]) self.ax_o = ng.make_axes([ ng.make_axis(name='C', length=K), ng.make_axis(name='D', length=M), ng.make_axis(name='H', length=P), ng.make_axis(name='W', length=Q), self.batch_axis ]) # Layer 1, using convolutation introduces multi/flatten view of tensors cf = ConvParams(C=2, N=4, K=1, H=2, W=2, R=2, S=2) inputs = ng.placeholder(axes=cf.ax_i) filters = ng.placeholder(axes=cf.ax_f) # randomly initialize from ngraph.testing import RandomTensorGenerator rng = RandomTensorGenerator(0, np.float32) # put value 1 into inputs/filters for conv input_value = rng.uniform(1, 1, cf.ax_i) filter_value = rng.uniform(1, 1, cf.ax_f) conv = ng.convolution(cf.conv_params, inputs, filters, axes=cf.ax_o) # Layer 2, using dot to ensure recv_op.axes == send_op.axes from ngraph.frontends.neon import UniformInit # put value 1 into weights for dot init_uni = UniformInit(1, 1) W_A = ng.make_axis(length=2) w_axes = ng.make_axes(W_A) + conv.axes.feature_axes() w = ng.variable(axes=w_axes, initial_value=init_uni) with ng.metadata(device_id='1'): dot = ng.dot(w, conv) with ExecutorFactory() as ex: dot_comp = ex.executor(dot, filters, inputs) dot_val = dot_comp(filter_value, input_value) np.testing.assert_array_equal(dot_val, [[8., 8., 8., 8.], [8., 8., 8., 8.]])
def Conv2D(self, tf_node, inputs): """ Computes a 2-D convolution given 4D input and filter tensors. Arguments: tf_node: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input, filter TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with maxpool Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image, weight = inputs # TODO: currently NHWC only assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC" # set axes shape ax_N = ng.make_axis(batch=True) ax_C = ng.make_axis(roles=[ar.Channel]) ax_D = ng.make_axis(roles=[ar.Depth]) ax_H = ng.make_axis(roles=[ar.Height]) ax_W = ng.make_axis(roles=[ar.Width]) ax_T = ng.make_axis(roles=[ar.Depth]) ax_R = ng.make_axis(roles=[ar.Height]) ax_S = ng.make_axis(roles=[ar.Width]) ax_K = ng.make_axis(roles=[ar.Channelout]) ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths) ng.make_axes([ax_R, ax_S, ax_C, ax_K]).set_shape(weight.axes.lengths) ax_D.length = 1 ax_T.length = 1 # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding( image.axes.lengths, weight.axes.lengths, tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # conv params params = dict(pad_d=0, pad_h=pad_t, pad_w=pad_l, str_d=1, str_h=str_h, str_w=str_w) # i, f, o axes ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]) ax_f = ng.make_axes([ax_C, ax_T, ax_R, ax_S, ax_K]) ax_o = ng.make_axes([ ng.make_axis(ax_K.length, name='C', roles=[ar.Channel]), spatial_axis(ax_i, ax_f, params['pad_d'], params['str_d'], ar.Depth), spatial_axis(ax_i, ax_f, params['pad_h'], params['str_h'], ar.Height), spatial_axis(ax_i, ax_f, params['pad_w'], params['str_w'], ar.Width), ax_N ]) # broadcast input / filter axes image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C])) image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN weight = ng.cast_axes(weight, ng.make_axes([ax_R, ax_S, ax_C, ax_K])) weight = ng.expand_dims(weight, ax_T, 0) # RSCK -> TRSCK weight = ng.axes_with_order(weight, axes=ax_f) # TRSCK -> CTRSK # convolution output = ng.convolution(params, image, weight, axes=ax_o) # cast back to NHWC oC, oD, oH, oW, oN = output.axes output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output = ng.Slice(output, out_slicing) return output
def Conv2D(self, tf_node, inputs): """ Computes a 2-D convolution given 4D input and filter tensors. Arguments: tf_node: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input, filter TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with maxpool Axes: Tensorflow Ngraph in (N, H, W, C) (C, D, H, W, N) filter (R, S, C, K) (C, T, R, S, K) out (N, P, Q, K) (K, M, P, Q, N) Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image, weight = inputs # TODO: currently NHWC only if tf_node.attr['data_format'].s.decode("ascii") != "NHWC": raise NotImplementedError("Only supports NHWC import for now.") # check in_C == f_C if image.axes.lengths[3] != weight.axes.lengths[2]: raise ValueError("Image's C dimension (%s) must be equal to " "filter's C dimension (%s)." % (image.axes.lengths[3], weight.axes.lengths[2])) # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = common_conv2d_pool_padding( image.axes.lengths, weight.axes.lengths, tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # conv params params = dict(pad_d=0, pad_h=pad_t, pad_w=pad_l, str_d=1, str_h=str_h, str_w=str_w, dil_d=1, dil_h=1, dil_w=1) # new axes C, D, H, W, T, R, S, K, M, P, Q = [ng.make_axis() for _ in range(11)] N = ng.make_axis(name='N') D.length, T.length, M.length = 1, 1, 1 # only supports 2D conv for now # tf's i, f, o axes ax_i_tf = ng.make_axes([N, H, W, C]) ax_f_tf = ng.make_axes([R, S, C, K]) ax_o_tf = ng.make_axes([N, P, Q, K]) ax_i_tf.set_shape(image.axes.lengths) ax_f_tf.set_shape(weight.axes.lengths) ax_o_tf.set_shape(common_conv2d_pool_output_shape(image.axes.lengths, weight.axes.lengths, tf_strides, padding)) # ngraph's i, f, o axes ax_i = ng.make_axes([C, D, H, W, N]) ax_f = ng.make_axes([C, T, R, S, K]) ax_o = ng.make_axes([K, M, P, Q, N]) # image NHWC -> CDHWN image = ng.cast_axes(image, ng.make_axes([N, H, W, C])) image = ng.expand_dims(image, D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, ax_i) # NDHWC -> CDHWN # weights RSCK -> CTRSK weight = ng.cast_axes(weight, ng.make_axes([R, S, C, K])) weight = ng.expand_dims(weight, T, 0) # RSCK -> TRSCK weight = ng.axes_with_order(weight, ax_f) # TRSCK -> CTRSK # convolution output = ng.convolution(params, image, weight, axes=ax_o) # output KMPQN -> NPQK # KMPQN -> NMPQK output = ng.axes_with_order(output, ng.make_axes([N, M, P, Q, K])) # NMPQK -> NPQK output = ng.tensor_slice(output, [slice(None), 0, slice(None), slice(None), slice(None)]) return output
def test_convolution_2d(): runtime = get_runtime() # input_x should have shape N(batch) x C x H x W input_x = ng.constant(np.array([ [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.], [0., 0., 5., 5., 0., 0., 0., 0., 0.]], dtype=np.float32).reshape(1, 1, 9, 9)) # filter weights should have shape M x C x kH x kW input_filter = ng.constant(np.array([ [1., 0., -1.], [2., 0., -2.], [1., 0., -1.]], dtype=np.float32).reshape(1, 1, 3, 3)) # convolution with padding=1 should produce 9 x 9 output: model = runtime.computation(ng.convolution(input_x, input_filter, padding_above=[1, 1], padding_below=[1, 1])) result = model() assert np.allclose(result, np.array([[[[0., -15., -15., 15., 15., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -20., -20., 20., 20., 0., 0., 0., 0.], [0., -15., -15., 15., 15., 0., 0., 0., 0.]]]], dtype=np.float32)) # convolution with padding=0 should produce 7 x 7 output: model = runtime.computation(ng.convolution(input_x, input_filter)) result = model() assert np.allclose(result, np.array([[[[-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0], [-20, -20, 20, 20, 0, 0, 0]]]], dtype=np.float32)) # convolution with strides=2 should produce 4 x 4 output: model = runtime.computation(ng.convolution(input_x, input_filter, filter_strides=[2, 2])) result = model() assert np.allclose(result, np.array([[[[-20., 20., 0., 0.], [-20., 20., 0., 0.], [-20., 20., 0., 0.], [-20., 20., 0., 0.]]]], dtype=np.float32)) # convolution with dilation=2 should produce 5 x 5 output: model = runtime.computation(ng.convolution(input_x, input_filter, filter_dilation_strides=(2, 2))) result = model() assert np.allclose(result, np.array([[[[0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0], [0, 0, 20, 20, 0]]]], dtype=np.float32))