def test_convolution(transformer_factory): """ test convolution forward path """ N = 128 C, K = 3, 8 D, T = 1, 1 H = W = 32 R = S = 2 padding = dict(pad_d=0, pad_h=0, pad_w=0) strides = dict(str_d=1, str_h=1, str_w=1) conv_params = padding.copy() conv_params.update(strides) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_f = ng.make_axes([ax.C, ax.T, ax.R, ax.S, ax.K]) ax_i.set_shape((C, D, H, W, N)) ax_f.set_shape((C, T, R, S, K)) ax_o = ng.make_axes([ ng.make_axis(ax_f.role_axes(ar.Channelout)[0].length, name='C', roles=[ar.Channel]), spatial_axis(ax_i, ax_f, padding['pad_d'], strides['str_d'], role=ar.Depth), spatial_axis(ax_i, ax_f, padding['pad_h'], strides['str_h'], role=ar.Height), spatial_axis(ax_i, ax_f, padding['pad_w'], strides['str_w'], role=ar.Width), ax.N ]) inputs = ng.placeholder(axes=ax_i) filters = ng.placeholder(axes=ax_f) # randomly initialize input_value = rng.uniform(-1, 1, ax_i) filter_value = rng.uniform(-1, 1, ax_f) assert input_value.shape == ax_i.lengths assert filter_value.shape == ax_f.lengths inputs = ng.placeholder(ax_i) filters = ng.placeholder(ax_f) output = ng.convolution(conv_params, inputs, filters, axes=ax_o) targets = ng.placeholder(axes=output.axes) costs = ng.cross_entropy_binary(ng.sigmoid(output), targets) error = ng.sum(costs, out_axes=()) / ng.batch_size(costs) d_inputs = ng.deriv(error, inputs) d_filters = ng.deriv(error, filters) targets_value = rng.uniform(.1, 0.9, output.axes) conv_executor = executor([output, error, d_inputs, d_filters], inputs, filters, targets) result_ng, err_ng, gradI_ng, gradF_ng = conv_executor( input_value, filter_value, targets_value) # Now compute reference values via NEON NervanaObject.be.bsz = N neon_layer = Convolution(fshape=(R, S, K), padding=padding, strides=strides) inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N)) neon_layer.W = neon_layer.be.array(filter_value.reshape(C * R * S * T, K)) neon_layer.dW = neon_layer.be.empty_like(neon_layer.W) neon_layer.configure((C, H, W)) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.set_deltas(DummyDeltaBuffers()) result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths) act_result_ne = 1. / (1.0 + np.exp(-result_ne)) err = neon_layer.be.array( (act_result_ne - targets_value).reshape(-1, N) / float(N)) gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths) gradF_ne = neon_layer.dW.get().reshape(ax_f.lengths) # Compare fprop np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6) # Compare bprop np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6) # Compare update np.testing.assert_allclose(gradF_ng, gradF_ne, rtol=0, atol=1e-4)
def Conv2D(self, tf_node, inputs): """ Computes a 2-D convolution given 4D input and filter tensors. Arguments: tf_node: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input, filter TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with maxpool Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image, weight = inputs # TODO: currently NHWC only assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC" # set axes shape ax_N = ng.make_axis(batch=True) ax_C = ng.make_axis(roles=[ar.Channel]) ax_D = ng.make_axis(roles=[ar.Depth]) ax_H = ng.make_axis(roles=[ar.Height]) ax_W = ng.make_axis(roles=[ar.Width]) ax_T = ng.make_axis(roles=[ar.Depth]) ax_R = ng.make_axis(roles=[ar.Height]) ax_S = ng.make_axis(roles=[ar.Width]) ax_K = ng.make_axis(roles=[ar.Channelout]) ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths) ng.make_axes([ax_R, ax_S, ax_C, ax_K]).set_shape(weight.axes.lengths) ax_D.length = 1 ax_T.length = 1 # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding( image.axes.lengths, weight.axes.lengths, tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # conv params params = dict(pad_d=0, pad_h=pad_t, pad_w=pad_l, str_d=1, str_h=str_h, str_w=str_w) # i, f, o axes ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]) ax_f = ng.make_axes([ax_C, ax_T, ax_R, ax_S, ax_K]) ax_o = ng.make_axes([ ng.make_axis(ax_K.length, name='C', roles=[ar.Channel]), spatial_axis(ax_i, ax_f, params['pad_d'], params['str_d'], ar.Depth), spatial_axis(ax_i, ax_f, params['pad_h'], params['str_h'], ar.Height), spatial_axis(ax_i, ax_f, params['pad_w'], params['str_w'], ar.Width), ax_N ]) # broadcast input / filter axes image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C])) image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN weight = ng.cast_axes(weight, ng.make_axes([ax_R, ax_S, ax_C, ax_K])) weight = ng.expand_dims(weight, ax_T, 0) # RSCK -> TRSCK weight = ng.axes_with_order(weight, axes=ax_f) # TRSCK -> CTRSK # convolution output = ng.convolution(params, image, weight, axes=ax_o) # cast back to NHWC oC, oD, oH, oW, oN = output.axes output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output = ng.Slice(output, out_slicing) return output
def MaxPool(self, tf_node, inputs): """ Performs the max pooling on the input. Arguments: tf_node: NodeDef object, the tensorflow node tso convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with conv2d Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image = inputs[0] # TODO: currently NHWC only assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC" # set axes shape ax_N = ng.make_axis(batch=True) ax_C = ng.make_axis(roles=[ar.Channel]) ax_D = ng.make_axis(roles=[ar.Depth]) ax_H = ng.make_axis(roles=[ar.Height]) ax_W = ng.make_axis(roles=[ar.Width]) ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths) ax_D.length = 1 # ksize params tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)] if len(tf_ksize) != 4: raise ValueError("Length of ksize my be 4.") if tf_ksize[0] != 1: raise NotImplementedError('Ksize on batch axis (N) must be 1.') if tf_ksize[3] != 1: raise NotImplementedError('Ksize on channel axis (C) must be 1.' 'Cross map pooling to be implemented.') R, S = tf_ksize[1:3] T = J = 1 # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding( image.axes.lengths, (R, S, ax_C.length, ax_C.length), tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # pooling params params = dict(op='max', pad_d=0, pad_h=pad_t, pad_w=pad_l, pad_c=0, str_d=1, str_h=str_h, str_w=str_w, str_c=1, J=J, T=T, R=R, S=S) # i, f, o axes ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]) ax_o = ng.make_axes([ spatial_axis(ax_i, J, params['pad_c'], params['str_c'], ar.Channel), spatial_axis(ax_i, T, params['pad_d'], params['str_d'], ar.Depth), spatial_axis(ax_i, R, params['pad_h'], params['str_h'], ar.Height), spatial_axis(ax_i, S, params['pad_w'], params['str_w'], ar.Width), ax_N ]) # broadcast input / filter axes image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C])) image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN # pooling output = ng.pooling(params, image, axes=ax_o) # cast back to NHWC oC, oD, oH, oW, oN = output.axes output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output = ng.Slice(output, out_slicing) return output
def test_pooling(): """ test pooling forward and backward path """ N = 128 C = 3 D = 1 H = W = 32 J = T = 1 R = S = 2 ngt.make_transformer() padding = dict(pad_d=0, pad_h=0, pad_w=0, pad_c=0) strides = dict(str_d=1, str_h=1, str_w=1, str_c=1) fshape = dict(J=J, T=T, R=R, S=S) pool_params = dict(op='max') pool_params.update(padding) pool_params.update(strides) pool_params.update(fshape) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_i.set_shape((C, D, H, W, N)) inputs = ng.placeholder(axes=ax_i) ax_o = ng.make_axes([ spatial_axis(ax_i, J, padding['pad_c'], strides['str_c'], role=ar.Channel), spatial_axis(ax_i, T, padding['pad_d'], strides['str_d'], role=ar.Depth), spatial_axis(ax_i, R, padding['pad_h'], strides['str_h'], role=ar.Height), spatial_axis(ax_i, S, padding['pad_w'], strides['str_w'], role=ar.Width), ax.N ]) # randomly initialize input_value = rng.uniform(-1, 1, ax_i) assert input_value.shape == ax_i.lengths # compute convolution with graph output = ng.pooling(pool_params, inputs, axes=ax_o) targets = ng.placeholder(axes=ax_o) costs = ng.cross_entropy_binary(ng.sigmoid(output), targets) error = ng.sum(costs, out_axes=()) / ng.batch_size(costs) d_inputs = ng.deriv(error, inputs) targets_value = rng.uniform(.1, 0.9, output.axes) conv_executor = executor([output, error, d_inputs], inputs, targets) result_ng, err_ng, gradI_ng = conv_executor(input_value, targets_value) # Now compute reference values via NEON NervanaObject.be.bsz = N neon_layer = Pooling(fshape=fshape, padding=padding, strides=strides, op="max") inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N)) neon_layer.configure((C, H, W)) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.set_deltas(DummyDeltaBuffers()) result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths) act_result_ne = 1. / (1.0 + np.exp(-result_ne)) err = neon_layer.be.array((act_result_ne - targets_value).reshape(-1, N) / float(N)) gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths) # Compare fprop np.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6) # Compare bprop np.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)