def test_wrong_op_name(): """ test wrong number of batch axes at input """ pf = PoolParams(op='min') inputs = ng.placeholder(axes=pf.ax_i) with pytest.raises(ValueError) as exinfo: ng.pooling(pf.pool_params, inputs, {}) assert str(exinfo.value) == "Unsupported pooling type: {pooltype}. Only max and avg " \ "pooling currently supported. ".format(pooltype=pf.pool_params['op'])
def test_wrong_input_shape_length(): """ test wrong input shape length """ ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W]) inputs = ng.placeholder(axes=ax_i) pool_params = dict(op='max') with pytest.raises(ValueError) as exinfo: ng.pooling(pool_params, inputs, {}) assert str(exinfo.value) == 'pooling input shape must be length 5, found {}' \ .format(len(ax_i))
def test_wrong_op_name(): """ test wrong number of batch axes at input """ ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) inputs = ng.placeholder(axes=ax_i) pooltype = 'min' pool_params = dict(op=pooltype) with pytest.raises(ValueError) as exinfo: ng.pooling(pool_params, inputs, {}) assert str(exinfo.value) == "Unsupported pooling type: {pooltype}. Only max and avg " \ "pooling currently supported. ".format(pooltype=pooltype)
def test_wrong_input_shape_length(): """ test wrong input shape length """ pf = PoolParams() ax_i = pf.ax_i[:-1] inputs = ng.placeholder(axes=ax_i) with pytest.raises(ValueError) as exinfo: ng.pooling(pf.pool_params, inputs, {}) assert str(exinfo.value) == 'pooling input shape must be length 5, found {}' \ .format(len(ax_i))
def train_outputs(self, in_obj): ppm = self.poolparams.copy() in_obj = ng.axes_with_role_order(in_obj, self.role_order) in_axes = in_obj.axes if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(roles=a.roles).named(a.short_name) for a in in_axes if not a.is_batch ]) # set lengths out_shape = [ output_dim(in_axes[0].length, ppm['J'], ppm['pad_d'], ppm['str_d']), output_dim(in_axes[1].length, ppm['T'], ppm['pad_d'], ppm['str_d']), output_dim(in_axes[2].length, ppm['R'], ppm['pad_h'], ppm['str_h']), output_dim(in_axes[3].length, ppm['S'], ppm['pad_w'], ppm['str_w']) ] self.o_axes.set_shape(out_shape) self.o_axes += in_axes.batch_axes() return ng.pooling(ppm, in_obj, axes=self.o_axes)
def train_outputs(self, in_obj): ppm = self.poolparams.copy() in_axes = in_obj.axes # TODO: clean this up if self.o_axes is None: self.o_axes = ng.make_axes([ ng.spatial_axis(in_axes, ppm['J'], ppm['pad_c'], ppm['str_c'], role=ar.Channel), ng.spatial_axis(in_axes, ppm['T'], ppm['pad_d'], ppm['str_d'], role=ar.Depth), ng.spatial_axis(in_axes, ppm['R'], ppm['pad_h'], ppm['str_h'], role=ar.Height), ng.spatial_axis(in_axes, ppm['S'], ppm['pad_w'], ppm['str_w'], role=ar.Width), ax.N ]) return ng.pooling(ppm, in_obj, axes=self.o_axes)
def test_wrong_number_of_batch_axes_at_input(): """ test wrong number of batch axes at input """ C = 3 D = 1 ax_C = ng.make_axis(name='N', length=C) ax_D = ng.make_axis(name='N', length=D) pool_params = dict(op='max') ax_i = ng.make_axes([ax_C, ax_D, ax.H, ax.W, ax.N]) inputs = ng.placeholder(axes=ax_i) with pytest.raises(ValueError) as exinfo: ng.pooling(pool_params, inputs, {}) assert str(exinfo.value) == "Input must have one batch axis. Found {n_batch_axes} batch" \ " axes: {batch_axes} and {n_sample_axes} sample axes: {sample_axes}.".format( n_batch_axes=len(inputs.axes.batch_axes()), batch_axes=inputs.axes.batch_axes(), n_sample_axes=len(inputs.axes.sample_axes()), sample_axes=inputs.axes.sample_axes())
def get_fprop_bprop(self, input_value): ip = ng.placeholder(axes=self.ax_i) ep = ng.placeholder(axes=self.ax_o) iv = np.array(input_value).astype(np.float32).reshape(self.dimI) ev = np.ones(self.dimO) * 4 output = ng.pooling(self.pool_params, ip, axes=self.ax_o) delta = BpropPoolOp(ep, ip, output) with executor([output, delta], ip, ep) as pool_executor: output_value, delta_value = pool_executor(iv, ev) return output_value, delta_value
def _pooling_op(self, cntk_op, inputs): """ Computes the pooling of a tensor. Arguments: cntk_op: CNTK function to be imported. inputs: List of inputs to this node. Returns: A ngraph Op. """ inputs = self._expand_input_axes(inputs[0]) C, D, H, W, N = inputs.axes M, P, Q = self._make_out_axes(cntk_op.shape) if cntk_op.attributes['poolingType'] == 0: pool_type = 'max' else: pool_type = 'avg' strides = self._make_strides(cntk_op.attributes['strides']) kernel = self._make_kernel(cntk_op.attributes['poolingWindowShape']) pad = self._make_padding('pool', cntk_op.attributes['autoPadding'], (D.length, H.length, W.length, C.length), kernel, (M.length, P.length, Q.length, C.length), strides) params = dict(op=pool_type, pad_d=pad[0], pad_h=pad[1], pad_w=pad[2], pad_c=pad[3], str_d=strides[0], str_h=strides[1], str_w=strides[2], str_c=strides[3], T=kernel[0], R=kernel[1], S=kernel[2], J=kernel[3]) pool = ng.pooling(params, inputs, [C, M, P, Q, N]) return squeeze_axes([pool])[0]
def _pooling_op(self, cntk_op, inputs, op): """ Computes the pooling of a tensor. CNTK Ngraph in ((C, H, W), N) (C, D, H, W, N) out (N, P, Q, K) (K, M, P, Q, N) Arguments: cntk_op: CNTK function to be imported. inputs: List of inputs to this node. op: 'max' for MaxPooling and 'avg' for AvgPooling Returns: A ngraph Op. """ inputs = self._expand_input_axes(inputs[0]) C, D, H, W, N = inputs.axes M, P, Q = self._make_out_axes(cntk_op.shape) strides = self._make_strides(cntk_op.attributes['strides']) kernel = self._make_kernel(cntk_op.attributes['poolingWindowShape']) pad = self._make_padding('pool', cntk_op.attributes['autoPadding'], (D.length, H.length, W.length, C.length), kernel, (M.length, P.length, Q.length, C.length), strides) params = dict(op=op, pad_d=pad[0], pad_h=pad[1], pad_w=pad[2], pad_c=pad[3], str_d=strides[0], str_h=strides[1], str_w=strides[2], str_c=strides[3], T=kernel[0], R=kernel[1], S=kernel[2], J=kernel[3]) pool = ng.pooling(params, inputs, [C, M, P, Q, N]) return remove_ones_axes([pool])[0]
def make_pooling_op(onnx_node, ng_inputs, custom_pool_params=None): # type: (NodeWrapper, List[TensorOp], Dict) -> Op """ Create an ngraph pooling Op based on an ONNX node. :param onnx_node: wrapped ONNX node for a pooling op :param ng_inputs: ngraph TensorOp input tensors :param custom_pool_params: optional pool_params overriding values based on onnx_node :return: ngraph pooling op """ x = ng_inputs[0] if len(x.axes) == 4: # 2D pooling # Reshape x axes from ONNX (N, C, H, W) to ngraph (C, D, H, W, N) x = reorder_axes(x, 'NCHW', 'CDHWN') elif len(x.axes) == 5: # 3D pooling # Reshape x axes from ONNX (N, C, H, W, D) to ngraph (C, D, H, W, N) x = reorder_axes(x, 'NCHWD', 'CDHWN') else: raise NotImplementedError( '%s node (%s): only 2D and 3D pooling ops are supported.', onnx_node.op_type, onnx_node.name) pool_params = get_pool_params(onnx_node) if custom_pool_params: pool_params.update(custom_pool_params) output_axes = make_pool_output_axes(x, pool_params) ng_op = ng.pooling(pool_params, x, output_axes) # ONNX output should have axes in the order N, C, H, W, D ng_op = reorder_axes(ng_op, 'CDHWN', 'NCHWD') if len(ng_inputs[0].axes ) == 4: # 2D convolution, slice away the D axis from output ng_op = ng.tensor_slice(ng_op, [ slice(None), slice(None), slice(None), slice(None), 0 ]) return ng_op
def __call__(self, in_obj): ppm = self.poolparams.copy() in_obj = reorder_spatial_axes(in_obj) in_axes = in_obj.axes if self.o_axes is None: self.o_axes = ng.make_axes([ ng.make_axis(name=a.name) for a in in_axes if not a.is_batch ]) # set lengths out_shape = [ output_dim(in_axes[0].length, ppm['J'], ppm['pad_d'], ppm['str_d']), output_dim(in_axes[1].length, ppm['T'], ppm['pad_d'], ppm['str_d']), output_dim(in_axes[2].length, ppm['R'], ppm['pad_h'], ppm['str_h']), output_dim(in_axes[3].length, ppm['S'], ppm['pad_w'], ppm['str_w']) ] self.o_axes.set_shape(out_shape) self.o_axes |= in_axes.batch_axis() return ng.pooling(ppm, in_obj, axes=self.o_axes)
def test_pooling(): """ test pooling forward and backward path """ N = 128 C = 3 D = 1 H = W = 32 J = T = 1 R = S = 2 ngt.make_transformer() padding = dict(pad_d=0, pad_h=0, pad_w=0, pad_c=0) strides = dict(str_d=1, str_h=1, str_w=1, str_c=1) fshape = dict(J=J, T=T, R=R, S=S) pool_params = dict(op='max') pool_params.update(padding) pool_params.update(strides) pool_params.update(fshape) ax_i = ng.make_axes([ax.C, ax.D, ax.H, ax.W, ax.N]) ax_i.set_shape((C, D, H, W, N)) inputs = ng.placeholder(axes=ax_i) ax_o = ng.make_axes([ ng.make_axis(roles=[ar.features_input]).named('C'), ng.make_axis(roles=[ar.features_0]).named('D'), ng.make_axis(roles=[ar.features_1]).named('H'), ng.make_axis(roles=[ar.features_2]).named('W'), ax.N ]) ax_o[:-1].set_shape((output_dim(C, J, padding['pad_c'], strides['str_c']), output_dim(D, T, padding['pad_d'], strides['str_d']), output_dim(H, R, padding['pad_h'], strides['str_h']), output_dim(W, S, padding['pad_w'], strides['str_w']))) # randomly initialize input_value = rng.uniform(-1, 1, ax_i) assert input_value.shape == ax_i.lengths # compute convolution with graph output = ng.pooling(pool_params, inputs, axes=ax_o) targets = ng.placeholder(axes=ax_o) costs = ng.cross_entropy_binary(ng.sigmoid(output), targets) error = ng.sum(costs, out_axes=()) / ng.batch_size(costs) d_inputs = ng.deriv(error, inputs) targets_value = rng.uniform(.1, 0.9, output.axes) with executor([output, error, d_inputs], inputs, targets) as conv_executor: result_ng, err_ng, gradI_ng = conv_executor(input_value, targets_value) # Now compute reference values via NEON NervanaObject.be.bsz = N neon_layer = Pooling(fshape=fshape, padding=padding, strides=strides, op="max") inp = neon_layer.be.array(input_value.reshape(C * H * W * D, N)) neon_layer.configure((C, H, W)) neon_layer.prev_layer = True neon_layer.allocate() neon_layer.set_deltas(DummyDeltaBuffers()) result_ne = neon_layer.fprop(inp).get().reshape(output.axes.lengths) act_result_ne = 1. / (1.0 + np.exp(-result_ne)) err = neon_layer.be.array( (act_result_ne - targets_value).reshape(-1, N) / float(N)) gradI_ne = neon_layer.bprop(err).get().reshape(ax_i.lengths) # Compare fprop ng.testing.assert_allclose(result_ng, result_ne, rtol=0, atol=1e-6) # Compare bprop ng.testing.assert_allclose(gradI_ng, gradI_ne, rtol=0, atol=1e-6)
def MaxPool(self, tf_node, inputs): """ Performs the max pooling on the input. Arguments: tf_node: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with conv2d Axes: Tensorflow Ngraph in (N, H, W, C) (C, D, H, W, N) out (N, P, Q, K) (K, M, P, Q, N) Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image = inputs[0] # TODO: currently NHWC only assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC" # new axes C, D, H, W, K, M, P, Q = [ng.make_axis() for _ in range(8)] N = ng.make_axis(name='N') D.length, M.length = 1, 1 # only supports 2D conv for now # tf's input axes ax_i_tf = ng.make_axes([N, H, W, C]) ax_i_tf.set_shape(image.axes.lengths) # ksize params tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)] if len(tf_ksize) != 4: raise ValueError("Length of ksize my be 4.") if tf_ksize[0] != 1: raise NotImplementedError('Ksize on batch axis (N) must be 1.') if tf_ksize[3] != 1: raise NotImplementedError('Ksize on channel axis (C) must be 1.' 'Cross map pooling to be implemented.') R_length, S_length = tf_ksize[1:3] T_length = J_length = 1 # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = common_conv2d_pool_padding( image.axes.lengths, (R_length, S_length, C.length, C.length), tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # pooling params params = dict(op='max', pad_d=0, pad_h=pad_t, pad_w=pad_l, pad_c=0, str_d=1, str_h=str_h, str_w=str_w, str_c=1, J=J_length, T=T_length, R=R_length, S=S_length) # tf's output axes ax_o_tf = ng.make_axes([N, P, Q, K]) ax_o_tf.set_shape(common_conv2d_pool_output_shape(image.axes.lengths, (R_length, S_length, C.length, C.length), tf_strides, padding)) # ngraph's i, f, o axes ax_i = ng.make_axes([C, D, H, W, N]) ax_o = ng.make_axes([K, M, P, Q, N]) # image NHWC -> CDHWN image = ng.cast_axes(image, ng.make_axes([N, H, W, C])) image = ng.expand_dims(image, D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, ax_i) # NDHWC -> CDHWN # pooling output = ng.pooling(params, image, axes=ax_o) # output KMPQN -> NPQK # KMPQN -> NMPQK output = ng.axes_with_order(output, ng.make_axes( [N, M, P, Q, K])) # NMPQK -> NPQK output = ng.tensor_slice(output, [slice(None), 0, slice(None), slice(None), slice(None)]) return output
def MaxPool(self, tf_node, inputs): """ Performs the max pooling on the input. Arguments: tf_node: NodeDef object, the tensorflow node tso convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the tensorflow node. Inputs to tf_node: input TODO: assume default tensorflow layout NHWC, RSCK, need to support NCHW as well need to clean up / merge with conv2d Notes on output shape: https://www.tensorflow.org/api_docs/python/nn.html#convolution """ image = inputs[0] # TODO: currently NHWC only assert tf_node.attr['data_format'].s.decode("ascii") == "NHWC" # set axes shape ax_N = ng.make_axis(batch=True) ax_C = ng.make_axis(roles=[ar.Channel]) ax_D = ng.make_axis(roles=[ar.Depth]) ax_H = ng.make_axis(roles=[ar.Height]) ax_W = ng.make_axis(roles=[ar.Width]) ng.make_axes([ax_N, ax_H, ax_W, ax_C]).set_shape(image.axes.lengths) ax_D.length = 1 # ksize params tf_ksize = [int(s) for s in list(tf_node.attr['ksize'].list.i)] if len(tf_ksize) != 4: raise ValueError("Length of ksize my be 4.") if tf_ksize[0] != 1: raise NotImplementedError('Ksize on batch axis (N) must be 1.') if tf_ksize[3] != 1: raise NotImplementedError('Ksize on channel axis (C) must be 1.' 'Cross map pooling to be implemented.') R, S = tf_ksize[1:3] T = J = 1 # strides params tf_strides = [int(s) for s in list(tf_node.attr['strides'].list.i)] if len(tf_strides) != 4: raise ValueError("Length of strides my be 4.") if tf_strides[0] != 1: raise NotImplementedError('Strides on batch axis (N) must be 1.') if tf_strides[3] != 1: raise NotImplementedError('Strides on channel axis (C) must be 1.') str_h, str_w = tf_strides[1], tf_strides[2] # padding params padding = tf_node.attr['padding'].s.decode("ascii") pad_t, pad_b, pad_l, pad_r = tf_conv2d_pool_padding( image.axes.lengths, (R, S, ax_C.length, ax_C.length), tf_strides, padding) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # pooling params params = dict(op='max', pad_d=0, pad_h=pad_t, pad_w=pad_l, pad_c=0, str_d=1, str_h=str_h, str_w=str_w, str_c=1, J=J, T=T, R=R, S=S) # i, f, o axes ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]) ax_o = ng.make_axes([ spatial_axis(ax_i, J, params['pad_c'], params['str_c'], ar.Channel), spatial_axis(ax_i, T, params['pad_d'], params['str_d'], ar.Depth), spatial_axis(ax_i, R, params['pad_h'], params['str_h'], ar.Height), spatial_axis(ax_i, S, params['pad_w'], params['str_w'], ar.Width), ax_N ]) # broadcast input / filter axes image = ng.cast_axes(image, ng.make_axes([ax_N, ax_H, ax_W, ax_C])) image = ng.expand_dims(image, ax_D, 1) # NHWC -> NDHWC image = ng.axes_with_order(image, axes=ax_i) # NDHWC -> CDHWN # pooling output = ng.pooling(params, image, axes=ax_o) # cast back to NHWC oC, oD, oH, oW, oN = output.axes output = ng.broadcast(output, ng.make_axes([oN, oD, oH, oW, oC])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output = ng.Slice(output, out_slicing) return output
def Pool(self, c2_op, inputs): """ Performs max or average pooling on the input. Arguments: c2_op: NodeDef object, the tensorflow node to convert. inputs: List of ngraph Ops as inputs to this node. Returns: A ngraph Op corresponding to the c2_op node. Inputs to c2_op: input """ supported_pooling = {'MaxPool': 'max', 'AveragePool': 'avg'} image = inputs[0] # TODO: we assume NCHW, make some assert here? # set input axes shape ax_N = ng.make_axis(name='N') ax_C = ng.make_axis() ax_D = ng.make_axis(length=1) ax_H = ng.make_axis() ax_W = ng.make_axis() ng.make_axes([ax_N, ax_C, ax_H, ax_W]).set_shape(image.axes.lengths) # create placeholders for output axes oC = ng.make_axis(name='C') oD = ng.make_axis(length=1, name='D') oH = ng.make_axis(name='H') oW = ng.make_axis(name='W') # spatial kernel size kernel_size = [int(val.i) for val in c2_op.arg if val.name == "kernel"] if len(kernel_size) != 1: raise ValueError("Kernel size must be scalar value") # kernel is square kernel_h = kernel_w = kernel_size[0] kernel_d = kernel_c = 1 # strides params stride_size = [int(val.i) for val in c2_op.arg if val.name == "stride"] if len(stride_size) != 1: raise ValueError("Stride size must be scalar value") stride_h = stride_w = stride_size[0] # padding params pad_t, pad_b, pad_l, pad_r = \ _c2_padding(c2_op, in_NHWC=[ax_N.length, ax_H.length, ax_W.length, ax_C.length], kernel_HWIO=[kernel_h, kernel_w, ax_C.length, ax_C.length], stride_NHWC=[1, stride_h, stride_w, 1]) if pad_t != pad_b or pad_l != pad_r: raise NotImplementedError("Requires symmetric padding in ngraph:" "pad_t(%s) == pad_b(%s) and" "pad_l(%s) == pad_r(%s)" % (pad_t, pad_b, pad_l, pad_r)) # pooling params params = dict(op=supported_pooling[c2_op.type], pad_d=0, pad_h=pad_t, pad_w=pad_l, pad_c=0, str_d=1, str_h=stride_h, str_w=stride_w, str_c=1, J=kernel_c, T=kernel_d, R=kernel_h, S=kernel_w) # i, o axes oC.length = output_dim(ax_C.length, kernel_c, params['pad_c'], params['str_c']) oD.length = output_dim(ax_D.length, kernel_d, params['pad_d'], params['str_d']) oH.length = output_dim(ax_H.length, kernel_h, params['pad_h'], params['str_h']) oW.length = output_dim(ax_W.length, kernel_w, params['pad_w'], params['str_w']) ax_i = ng.make_axes([ax_C, ax_D, ax_H, ax_W, ax_N]) ax_o = ng.make_axes([oC, oD, oH, oW, ax_N]) # broadcast input / filter axes image = ng.cast_axes(image, ng.make_axes([ax_N, ax_C, ax_H, ax_W])) image = ng.expand_dims(image, ax_D, 1) # NCHW -> NDCHW image = ng.axes_with_order(image, axes=ax_i) # NDCHW -> CDHWN # pooling output = ng.pooling(params, image, axes=ax_o) # cast back to NDCHW output = ng.broadcast(output, ng.make_axes([ax_N, oD, oC, oH, oW])) # slice away the oD out_slicing = [slice(None), 0, slice(None), slice(None), slice(None)] output = ng.tensor_slice(output, out_slicing) return output