def _Conv2DGrad(op, grad): """Weight sharing for symmetric lateral connections.""" strides = op.get_attr('strides') padding = op.get_attr('padding') use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu') data_format = op.get_attr('data_format') shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) dx = nn_ops.conv2d_backprop_input(shape_0, op.inputs[1], grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw = nn_ops.conv2d_backprop_filter(op.inputs[0], shape_1, grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw_t = tf.transpose(dw, (2, 3, 0, 1)) dw_symm_t = (0.5) * (dw_t + tf.transpose(dw_t, (1, 0, 2, 3))) dw_symm = tf.transpose(dw_symm_t, (2, 3, 0, 1)) return dx, dw_symm
def testConvBackpropFilter(self): with self.session() as sess: with ops.device("/device:IPU:0"): inp = array_ops.placeholder(np.float32, [2, 8, 8, 3]) fil = constant_op.constant([2, 2, 3, 5], np.int32) bck = array_ops.placeholder(np.float32, [2, 8, 8, 5], name="wei") output = nn_ops.conv2d_backprop_filter(inp, fil, bck, strides=[1, 1, 1, 1], padding="SAME") report = tu.ReportJSON(self, sess) report.reset() fd = { inp: np.zeros([2, 8, 8, 3]), bck: np.zeros([2, 8, 8, 5]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([2, 2, 3, 5])) report.parse_log() ok = [ '__seed*', 'Copy_', 'Conv2DBackpropFilter/convolution.*/Conv_8x8', ] report.assert_all_compute_sets_and_list(ok)
def _Conv2DGrad(op, grad): strides = op.get_attr("strides") padding = op.get_attr("padding") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") data_format = op.get_attr("data_format") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) dx = nn_ops.conv2d_backprop_input( shape_0, op.inputs[1], grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw = nn_ops.conv2d_backprop_filter( op.inputs[0], shape_1, grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw_t = tf.transpose(dw,(2,3,0,1)) dw_symm_t = (0.5) * (dw_t + tf.transpose(dw_t,(1,0,2,3))) dw_symm = tf.transpose(dw_symm_t,(2,3,0,1)) return dx,dw_symm
def conv_scaled_inplace(input_values, grads, weights, lr): return weights - lr * nn_ops.conv2d_backprop_filter( input_values, filter_sizes, grads, strides=[1, 1, 1, 1], padding="SAME")
def testGradient(self): with self.cached_session(): for padding in ["SAME", "VALID"]: for stride in [1, 2]: np.random.seed(1) in_shape = [5, 8, 6, 4] in_val = constant_op.constant( 2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32) filter_shape = [3, 3, 4, 6] # Make a convolution op with the current settings, just to easily get # the shape of the output. conv_out = nn_ops.conv2d(in_val, array_ops.zeros(filter_shape), strides=[1, stride, stride, 1], padding=padding) out_backprop_shape = conv_out.get_shape().as_list() out_backprop_val = constant_op.constant( 2 * np.random.random_sample(out_backprop_shape) - 1, dtype=dtypes.float32) output = nn_ops.conv2d_backprop_filter( in_val, filter_shape, out_backprop_val, strides=[1, stride, stride, 1], padding=padding) err = gradient_checker.compute_gradient_error( [in_val, out_backprop_val], [in_shape, out_backprop_shape], output, filter_shape) print("conv2d_backprop_filter gradient err = %g " % err) err_tolerance = 2e-3 self.assertLess(err, err_tolerance)
def _MpuSimConv2DGrad(op, grad): """Gradient function for MpuSimConv2D.""" dilations = op.get_attr("dilations") strides = op.get_attr("strides") padding = op.get_attr("padding") data_format = op.get_attr("data_format") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) # We call the gen_nn_ops backprop functions instead of nn_ops backprop # functions for performance reasons in Eager mode. gen_nn_ops functions take a # `explicit_paddings` parameter, but nn_ops functions do not. So if were were # to use the nn_ops functions, we would have to convert `padding` and # `explicit_paddings` into a single `padding` parameter, increasing overhead # in Eager mode. return [ nn_ops.conv2d_backprop_input( shape_0, op.inputs[1], grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=False, data_format=data_format), nn_ops.conv2d_backprop_filter( op.inputs[0], shape_1, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=False, data_format=data_format) ]
def _Conv2DBackpropInputGrad(op, grad): """The derivatives for deconvolution. Args: op: the Deconvolution op. grad: the tensor representing the gradient w.r.t. the output Returns: the gradients w.r.t. the input and the filter """ return [ None, nn_ops.conv2d_backprop_filter( grad, array_ops.shape(op.inputs[1]), op.inputs[2], dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), data_format=op.get_attr("data_format").decode()), nn_ops.conv2d( grad, op.inputs[1], dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), data_format=op.get_attr("data_format").decode()) ]
def _RunAndVerifyBackpropFilter(self, input_sizes, filter_sizes, output_sizes, strides, padding, expected, data_format, use_ve): total_input_size = 1 total_output_size = 1 for s in input_sizes: total_input_size *= s for s in output_sizes: total_output_size *= s # Initializes the input tensor with array containing incrementing # numbers from 1. x0 = [f * 1.0 for f in range(1, total_input_size + 1)] x2 = [f * 1.0 for f in range(1, total_output_size + 1)] for dtype in self._DtypesToTest(use_ve=use_ve): with test_util.device(use_ve=use_ve, use_gpu=False): t0 = constant_op.constant(x0, shape=input_sizes, dtype=dtype) t1 = constant_op.constant(filter_sizes, shape=[len(filter_sizes)]) t2 = constant_op.constant(x2, shape=output_sizes, dtype=dtype) explicit_strides = [1] + strides + [1] if data_format == "NCHW": t0 = test_util.NHWCToNCHW(t0) t2 = test_util.NHWCToNCHW(t2) explicit_strides = test_util.NHWCToNCHW(explicit_strides) conv = nn_ops.conv2d_backprop_filter(t0, t1, t2, strides=explicit_strides, padding=padding, data_format=data_format) value = self.evaluate(conv) self.assertShapeEqual(value, conv) tf_logging.info("expected = ", expected) tf_logging.info("actual = ", value) self.assertArrayNear(expected, value.flatten(), 1e-5)
def tf_model(padding): t1 = tf.compat.v1.placeholder(dtype=tf.float32, shape=input_sizes_nhwc, name='t1') t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t2') t3 = tf.compat.v1.placeholder(dtype=tf.float32, shape=out_backprop_in_sizes[padding], name='t3') #reshaping the out_backprop to NHWC since TF does not support NCHW t3 = tf.transpose(t3, [0, 2, 3, 1]) #Cast dtype to bfloat16 for TF because NNP casts ng_model inputs t1 = tf.cast(t1, dtype=tf.bfloat16) t3 = tf.cast(t3, dtype=tf.bfloat16) filt = nn_ops.conv2d_backprop_filter(t1, t2, t3, stride_nhwc, padding=padding, data_format='NHWC') #Cast dtype back to float32 similar to NNP filt = tf.cast(filt, dtype=tf.float32) return filt, t1, t3
def _PerturbConv2DGrad(op, grad): """Set grads for the middle-most column to 0.""" strides = op.get_attr('strides') padding = op.get_attr('padding') use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu') data_format = op.get_attr('data_format') shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) dx = nn_ops.conv2d_backprop_input(shape_0, op.inputs[1], grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw = nn_ops.conv2d_backprop_filter(op.inputs[0], shape_1, grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) # # Find middle unit # h, w = shape_1[1] // 2, shape_1[2] // 2 # # Set middle unit gradient to 0 # dx[:, h, w] = 0. # Find middle unit of hidden state (these are weights) h, w = shape_0[1] // 2, shape_0[2] // 2 # Set middle unit gradient to 0 dw[:, h, w] = 0. return dx, dw
def _Conv2DGrad(op, grad): dilations = op.get_attr("dilations") strides = op.get_attr("strides") padding = op.get_attr("padding") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") data_format = op.get_attr("data_format") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) return [ nn_ops.conv2d_backprop_input( shape_0, op.inputs[1], grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format), nn_ops.conv2d_backprop_filter( op.inputs[0], shape_1, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) ]
def _Conv2DBackpropInputGrad(op, grad): """The derivatives for deconvolution. Args: op: the Deconvolution op. grad: the tensor representing the gradient w.r.t. the output Returns: the gradients w.r.t. the input and the filter """ return [ None, nn_ops.conv2d_backprop_filter( grad, array_ops.shape(op.inputs[1]), op.inputs[2], dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), data_format=op.get_attr("data_format")), nn_ops.conv2d(grad, op.inputs[1], dilations=op.get_attr("dilations"), strides=op.get_attr("strides"), padding=op.get_attr("padding"), use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"), data_format=op.get_attr("data_format")) ]
def testGradientDilatedConv(self): if test.is_gpu_available(cuda_only=True): with self.test_session(use_gpu=True): for padding in ["SAME", "VALID"]: for stride in [1, 2]: np.random.seed(1) in_shape = [5, 8, 6, 4] in_val = constant_op.constant( 2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32) filter_shape = [3, 3, 4, 6] # Make a convolution op with the current settings, # just to easily get the shape of the output. conv_out = nn_ops.conv2d( in_val, array_ops.zeros(filter_shape), dilations=[1, 2, 2, 1], strides=[1, stride, stride, 1], padding=padding) out_backprop_shape = conv_out.get_shape().as_list() out_backprop_val = constant_op.constant( 2 * np.random.random_sample(out_backprop_shape) - 1, dtype=dtypes.float32) output = nn_ops.conv2d_backprop_filter( in_val, filter_shape, out_backprop_val, dilations=[1, 2, 2, 1], strides=[1, stride, stride, 1], padding=padding) err = gradient_checker.compute_gradient_error( [in_val, out_backprop_val], [in_shape, out_backprop_shape], output, filter_shape) print("conv2d_backprop_filter gradient err = %g " % err) err_tolerance = 2e-3 self.assertLess(err, err_tolerance)
def _Conv2DGrad(op, grad): dilations = op.get_attr("dilations") strides = op.get_attr("strides") padding = op.get_attr("padding") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") data_format = op.get_attr("data_format") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) return [ nn_ops.conv2d_backprop_input(shape_0, op.inputs[1], grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format), nn_ops.conv2d_backprop_filter(op.inputs[0], shape_1, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) ]
def testConvBackpropFilter(self): with ops.device("/device:IPU:0"): inp = array_ops.placeholder(np.float32, [2, 8, 8, 3]) fil = constant_op.constant([2, 2, 3, 5], np.int32) bck = array_ops.placeholder(np.float32, [2, 8, 8, 5], name="wei") output = nn_ops.conv2d_backprop_filter( inp, fil, bck, strides=[1, 1, 1, 1], padding="SAME") with ops.device('cpu'): report = gen_ipu_ops.ipu_event_trace() tu.configure_ipu_system() with tu.ipu_session() as sess: sess.run(report) fd = { inp: np.zeros([2, 8, 8, 3]), bck: np.zeros([2, 8, 8, 5]), } result = sess.run(output, fd) self.assertAllClose(result, np.zeros([2, 2, 3, 5])) result = sess.run(report) s = tu.extract_all_strings_from_event_trace(result) cs_list = tu.get_compute_sets_from_report(s) ok = ['__seed*', 'Copy_', 'Conv2DBackpropFilter/convolution.*/Conv_8x8'] self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def _Conv2DGrad(op, grad): return [ nn_ops.conv2d_backprop_input( array_ops.shape(op.inputs[0]), op.inputs[1], grad, op.get_attr("strides"), op.get_attr("padding") ), nn_ops.conv2d_backprop_filter( op.inputs[0], array_ops.shape(op.inputs[1]), grad, op.get_attr("strides"), op.get_attr("padding") ), ]
def testBackwardFilterGradient(self): np.random.seed(1) in_shape = LayerShape(batch=8, height=128, width=128, channels=8) filter_shape = FilterShape(height=3, width=3, in_channels=8, out_channels=8) in_op = self._random_data_op(in_shape) out_op = self._random_out_op(in_shape, filter_shape) filter_gradient_op = nn_ops.conv2d_backprop_filter( in_op, filter_shape, out_op, strides=_STRIDES, padding=_PADDING) self._assert_reproducible(filter_gradient_op)
def _Conv2DGrad(op, grad): return [nn_ops.conv2d_backprop_input( array_ops.shape(op.inputs[0]), op.inputs[1], grad, op.get_attr("strides"), op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"), op.get_attr("data_format")), nn_ops.conv2d_backprop_filter(op.inputs[0], array_ops.shape(op.inputs[1]), grad, op.get_attr("strides"), op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"), op.get_attr("data_format"))]
def testBackwardFilterGradient(self): in_shape = LayerShapeNHWC(batch=8, height=128, width=128, channels=8) filter_shape = FilterShape2D( height=3, width=3, in_channels=8, out_channels=8) in_op = self._random_data_op(in_shape) strides = [1, 1, 1, 1] padding = 'SAME' out_op = self._random_out_op(in_shape, filter_shape, strides, padding) filter_gradient_op = nn_ops.conv2d_backprop_filter( in_op, filter_shape, out_op, strides=strides, padding=padding) self._assert_reproducible(filter_gradient_op)
def conv_backprop_filter_tf(inputmats, filter_sizes, out_backprop, stride, padding='VALID'): strd = [1, stride, stride, 1] d_w_backprop_filter = nn_ops.conv2d_backprop_filter( input=inputmats, filter_sizes=filter_sizes, out_backprop=out_backprop, strides=strd, padding=padding) return np.array(d_w_backprop_filter)
def ng_model(padding): t1 = tf.placeholder(dtype=tf.float32, shape=input_sizes_nhwc, name='t1') t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t1') t3 = tf.placeholder(dtype=tf.float32, shape=out_backprop_in_sizes[padding], name='t3') filt = nn_ops.conv2d_backprop_filter(t1, t2, t3, stride, padding=padding, data_format='NHWC') return filt, t1, t3
def testConvBackwardFilterGradient(self, rate=1): in_shape = LayerShapeNHWC(batch=8, height=64, width=64, channels=8) filter_shape = FilterShape2D( height=3, width=3, in_channels=8, out_channels=8) in_op = self._random_data_op(in_shape) strides = [1, 1, 1, 1] padding = 'SAME' dilations = [1, rate, rate, 1] out_op = self._random_out_op(in_shape, filter_shape, strides, padding, dilations) self._assert_reproducible(lambda: nn_ops.conv2d_backprop_filter( in_op, filter_shape, out_op, strides=strides, padding=padding, dilations=dilations))
def _DeConv2DGrad(op, grad): """The derivatives for deconvolution. Args: op: the Deconvolution op. grad: the tensor representing the gradient w.r.t. the output Returns: the gradients w.r.t. the input and the filter """ return [ None, nn_ops.conv2d_backprop_filter( grad, array_ops.shape(op.inputs[1]), op.inputs[2], op.get_attr("strides"), op.get_attr("padding") ), nn_ops.conv2d(grad, op.inputs[1], op.get_attr("strides"), op.get_attr("padding")), ]
def _DeConv2DGrad(op, grad): """The derivatives for deconvolution. Args: op: the Deconvolution op. grad: the tensor representing the gradient w.r.t. the output Returns: the gradients w.r.t. the input and the filter """ return [ None, nn_ops.conv2d_backprop_filter(grad, array_ops.shape(op.inputs[1]), op.inputs[2], op.get_attr("strides"), op.get_attr("padding")), nn_ops.conv2d(grad, op.inputs[1], op.get_attr("strides"), op.get_attr("padding")) ]
def testGradientDilatedConv(self): if test.is_gpu_available(cuda_only=True): with self.session(): for padding in [ "SAME", "VALID", [(0, 0), (3, 5), (2, 1), (0, 0)], [(0, 0), (5, 2), (5, 1), (0, 0)] ]: for stride in [1, 2]: np.random.seed(1) in_shape = [5, 8, 6, 4] in_val = constant_op.constant( 2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32) filter_shape = [3, 3, 4, 6] # Make a convolution op with the current settings, # just to easily get the shape of the output. conv_out = nn_ops.conv2d( in_val, array_ops.zeros(filter_shape), dilations=[1, 2, 2, 1], strides=[1, stride, stride, 1], padding=padding) out_backprop_shape = conv_out.get_shape().as_list() out_backprop_val = constant_op.constant( 2 * np.random.random_sample(out_backprop_shape) - 1, dtype=dtypes.float32) output = nn_ops.conv2d_backprop_filter( in_val, filter_shape, out_backprop_val, dilations=[1, 2, 2, 1], strides=[1, stride, stride, 1], padding=padding) err = gradient_checker.compute_gradient_error( [in_val, out_backprop_val], [in_shape, out_backprop_shape], output, filter_shape) print("conv2d_backprop_filter gradient err = %g " % err) err_tolerance = 1e-2 self.assertLess(err, err_tolerance)
def _Conv2DGrad(op, grad): return [ nn_ops.conv2d_backprop_input( array_ops.shape(op.inputs[0]), op.inputs[1], grad, op.get_attr("strides"), op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"), op.get_attr("data_format"), ), nn_ops.conv2d_backprop_filter( op.inputs[0], array_ops.shape(op.inputs[1]), grad, op.get_attr("strides"), op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"), op.get_attr("data_format"), ), ]
def tf_model(padding): t1 = tf.placeholder(dtype=tf.float32, shape=input_sizes_nhwc, name='t1') t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t1') t3 = tf.placeholder(dtype=tf.float32, shape=out_backprop_in_sizes[padding], name='t3') #Cast dtype to bfloat16 for TF because NNP casts ng_model inputs t1 = tf.cast(t1, dtype=tf.bfloat16) t3 = tf.cast(t3, dtype=tf.bfloat16) filt = nn_ops.conv2d_backprop_filter(t1, t2, t3, stride, padding=padding, data_format='NHWC') #Cast dtype back to float32 similar to NNP filt = tf.cast(filt, dtype=tf.float32) return filt, t1, t3
def _SConv2DGrad(op, grad): """Weight sharing for symmetric lateral connections.""" strides = op.get_attr('strides') padding = op.get_attr('padding') use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu') data_format = op.get_attr('data_format') shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) dx = nn_ops.conv2d_backprop_input(shape_0, op.inputs[1], grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw = nn_ops.conv2d_backprop_filter(op.inputs[0], shape_1, grad, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) dw = 0.5 * (dw + dw[::-1, ::-1, :, :]) return dx, dw
def test_conv2d_grad_weight(): ''' Run tests on the Wave custom conv2d operator. ''' tf.reset_default_graph() # Turn off graph-rewriting optimizations config = tf.ConfigProto(graph_options=tf.GraphOptions( optimizer_options=tf.OptimizerOptions( opt_level=tf.OptimizerOptions.L0))) # NCHW t_n = 1 t_ci = 2 t_co = 4 t_h = 4 t_w = t_h w_k = 3 t_ho = t_h - (w_k - 1) t_wo = t_w - (w_k - 1) if True: # N H W C gradient_same = tf.get_variable( "gs", [t_n, t_h, t_w, t_co], dtype=tf.float32, initializer=tf.glorot_normal_initializer()) gradient_valid = tf.get_variable( "gv", [t_n, t_ho, t_wo, t_co], dtype=tf.float32, initializer=tf.glorot_normal_initializer()) # K K Ci Co c2d_in = tf.get_variable("f", [t_n, t_h, t_w, t_ci], dtype=tf.float32, initializer=tf.glorot_normal_initializer()) total_iters = 100 else: total_iters = 100 ''' # N H W C gradient_same = tf.get_variable("gs", dtype=tf.float32, initializer=[[[[ 0.20130304, 0.27111197, -0.23086302, -0.23035139], [-0.09796654, -0.25246394, -0.37301826, -0.04510078], [ 0.23488244, 0.11107023, -0.06509414, -0.10155579], [-0.02653619, 0.25628388, 0.14519225, -0.03400368]], [[-0.2737653, -0.09805538, 0.2395286, -0.05114909], [ 0.25615498, -0.00413944, -0.11665037, 0.41589662], [-0.2739116, -0.25920913, -0.09962016, 0.09206288], [ 0.20151283, 0.17594334, -0.07801611, 0.18326335]], [[ 0.41815385, -0.2302951, 0.04820603, 0.28555122], [-0.07512673, 0.03257893, 0.26206508, -0.23212446], [ 0.11086888, -0.14072362, 0.0384812, -0.3119425 ], [ 0.12364136, 0.20889276, 0.21803159, 0.23013732]], [[ 0.14886299, 0.2521119, 0.09665762, -0.0950445 ], [-0.02764839, -0.22908632, -0.21170329, -0.01173863], [-0.01734301, -0.09034941, -0.36093128, -0.10678421], [ 0.22199751, -0.07065484, -0.20354767, 0.35294214]]]]) # K K Ci Co c2d_in = tf.get_variable("f", dtype=tf.float32, initializer=[[[[ 0.05964299, -0.05988665], [ 0.1176498, -0.03819368], [ 0.5281046, -0.23744191], [ 0.11645658, -0.19929294]], [[-0.23003536, 0.00503353], [-0.13039683, 0.07515113], [ 0.27454942, 0.07771762], [-0.30660862, 0.0843183 ]], [[ 0.45593834, -0.10620885], [-0.2692576, -0.21969746], [-0.0423032, -0.27153978], [-0.12364867, 0.4992814 ]], [[ 0.199746, -0.36852396], [ 0.06857206, -0.5132366 ], [-0.3468758, -0.11179192], [-0.00362369, 0.24938436]]]]) ''' # N H W C gradient_valid = tf.get_variable( "gv", dtype=tf.float32, initializer=[[[[-0.46859148, -0.47966853, 0.20002109, -0.56815886], [0.28490868, -0.34714422, -0.02198983, -0.16817461]], [[0.55926424, 0.43453974, -0.14650254, -0.35478827], [0.33911255, -0.13839267, 0.50347936, -0.24987353]]]]) # K K Ci Co c2d_in = tf.get_variable("f", dtype=tf.float32, initializer=[[[[-0.02360273, 0.11748651], [-0.06789866, -0.09688393], [0.19889985, -0.06614035], [0.1964069, -0.49319836]], [[-0.11062168, 0.10862893], [-0.11725109, -0.25628847], [-0.03851736, -0.04931202], [0.21630755, -0.22763665]], [[0.22559226, 0.38029733], [0.10525792, -0.36071476], [0.05696308, -0.12789722], [0.07823437, 0.0835037]], [[0.2075723, 0.14272317], [0.50318813, 0.3311627], [-0.0821431, 0.15861501], [-0.0524208, -0.01618423]]]]) wgt_shape = tf.get_variable("s", dtype=tf.int32, initializer=[w_k, w_k, t_ci, t_co]) t_init = tf.global_variables_initializer() for i in range(total_iters): # SAME variant with tf.Session('', config=config) as sess: t_init.run() # print("Wave Kernel:\n-------------------------------------------------") #z_op = nn_ops.conv2d_backprop_filter( z_op = waveflow.wavecomp_ops_module.wave_conv2d_gradient_weight( c2d_in, wgt_shape, gradient_same, strides=[1, 1, 1, 1], padding='SAME', use_cudnn_on_gpu=False, data_format='NHWC') # Base tensorflow. Only supports NHWC. z2_op = nn_ops.conv2d_backprop_filter( #z2_op = wavecomp_ops_module.wave_conv2d_gradient_weight( c2d_in, wgt_shape, gradient_same, strides=[1, 1, 1, 1], padding='SAME', use_cudnn_on_gpu=False, data_format='NHWC') # z = z_op.eval() # z2 = z2_op.eval() z, z2, grad_val, in_val = sess.run( [z_op, z2_op, gradient_same, c2d_in]) # print("\nTF:\n-------------------------------------------------") assert_str = "Failure on i: %d, mode: SAME" % (i) if not compare_tensor(z, z2, assert_str): print("gradients: %s" % (grad_val)) print("c2d_in: %s" % (in_val)) print("z: shape: %s, %s" % (z.shape, z)) print("z (np): shape: %s, %s" % (z2.shape, z2)) print("\n\n") assert False # Valid variant with tf.Session('', config=config) as sess: t_init.run() # print("Wave Kernel:\n-------------------------------------------------") #z_op = nn_ops.conv2d_backprop_filter( z_op = waveflow.wavecomp_ops_module.wave_conv2d_gradient_weight( c2d_in, wgt_shape, gradient_valid, strides=[1, 1, 1, 1], padding='VALID', use_cudnn_on_gpu=False, data_format='NHWC') # Base tensorflow. Only supports NHWC. z2_op = nn_ops.conv2d_backprop_filter( #z2_op = wavecomp_ops_module.wave_conv2d_gradient_weight( c2d_in, wgt_shape, gradient_valid, strides=[1, 1, 1, 1], padding='VALID', use_cudnn_on_gpu=False, data_format='NHWC') z, z2, grad_val, in_val = sess.run( [z_op, z2_op, gradient_valid, c2d_in]) # print("\nTF:\n-------------------------------------------------") assert_str = "Failure on i: %d, mode: VALID" % (i) if not compare_tensor(z, z2, assert_str): print("gradients: %s" % (grad_val)) print("c2d_in: %s" % (in_val)) print("z: shape: %s, %s" % (z.shape, z)) print("z (np): shape: %s, %s" % (z2.shape, z2)) print("\n\n") assert False return True
def _Conv2DGrad(op, grad): """Gradient function for Conv2D.""" dilations = op.get_attr("dilations") strides = op.get_attr("strides") padding = op.get_attr("padding") explicit_paddings = op.get_attr("explicit_paddings") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") data_format = op.get_attr("data_format") use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu") shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]]) enable_quantop_grad = int(os.getenv('ENABLE_QUANTOP_CONV_GRAD', 0)) enable_quantop_input = int(os.getenv('ENABLE_QUANTOP_CONV', 0)) enable_quantop_wtgrad = int(os.getenv('ENABLE_QUANTOP_CONV_WTGRAD', 0)) dformat = 'channels_last' inp_channels = op.inputs[0].get_shape()[3].value if data_format == b'NCHW': dformat = 'channels_first' inp_channels = op.inputs[0].get_shape()[1].value elif data_format == b'None': dformat = 'unknown' quant_input_copy = int(os.getenv('QUANTEMU_ALLOCATE_COPY_INPUTS', 23)) quant_filter_copy = int(os.getenv('QUANTEMU_ALLOCATE_COPY_FILTERS', 23)) quant_input_precision = int(os.getenv('QUANTEMU_PRECISION_CONV_INPUTS', 23)) quant_filter_precision = int( os.getenv('QUANTEMU_PRECISION_CONV_FILTERS', 23)) quant_grad_precision = int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23)) quant_wtgrad_precision = int( os.getenv('QUANTEMU_PRECISION_CONV_WTGRADS', 23)) if inp_channels == 3: quant_grad_precision = quant_input_precision = int( os.getenv('QUANTEMU_FIRST_LAYER_PRECISION', 23)) quant_filter_precision = int( os.getenv('QUANTEMU_FIRST_LAYER_PRECISION', 23)) if enable_quantop_grad == 1: grad = quantemu_ops.quantize_emu( grad, data_format=dformat, data_type=int(os.getenv('QUANTEMU_GRAD_DATA_TYPE', 0)), precision= quant_grad_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23)), exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)), channel_blocking_type=int( os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_GRADS', 0)), channels_per_block=int(os.getenv('QUANTEMU_CBLOCK_SIZE_GRADS', 0)), round_mode=int(os.getenv('QUANTEMU_RMODE_GRADS', 0))) if enable_quantop_input == 1: if quant_input_copy == 1: acts = quantemu_ops.quantize_emu( op.inputs[0], data_format=dformat, data_type=int(os.getenv('QUANTEMU_INPUT_DATA_TYPE', 0)), precision= quant_input_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_INPUTS', 23)), exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)), channel_blocking_type=int( os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_INPUTS', 0)), channels_per_block=int( os.getenv('QUANTEMU_CBLOCK_SIZE_INPUTS', 0)), round_mode=int(os.getenv('QUANTEMU_RMODE_INPUTS', 0))) else: acts = op.inputs[0] if quant_filter_copy == 1: filters = quantemu_ops.quantize_emu( op.inputs[1], data_format=dformat, allocate_copy=int( os.getenv('QUANTEMU_ALLOCATE_COPY_FILTERS', 0)), data_type=int(os.getenv('QUANTEMU_FILTER_DATA_TYPE', 0)), precision= quant_filter_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_FILTERS', 23)), exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)), channel_blocking_type=int( os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_FILTERS', 0)), channels_per_block=int( os.getenv('QUANTEMU_CBLOCK_SIZE_FILTERS', 0)), round_mode=int(os.getenv('QUANTEMU_RMODE_FILTERS', 0))) else: filters = op.inputs[1] outgrad = nn_ops.conv2d_backprop_input( shape_0, #op.inputs[1], filters, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) wtgrad = nn_ops.conv2d_backprop_filter( #op.inputs[0], acts, shape_1, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) if enable_quantop_wtgrad == 1: wtgrad = quantemu_ops.quantize_emu( wtgrad, data_format=dformat, data_type=int(os.getenv('QUANTEMU_WTGRAD_DATA_TYPE', 0)), precision= quant_wtgrad_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23)), exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)), channel_blocking_type=int( os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_WTGRADS', 0)), channels_per_block=int( os.getenv('QUANTEMU_CBLOCK_SIZE_WTGRADS', 0)), round_mode=int(os.getenv('QUANTEMU_RMODE_WTGRADS', 0))) return [outgrad, wtgrad] else: # No Quantization return [ nn_ops.conv2d_backprop_input(shape_0, op.inputs[1], grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format), nn_ops.conv2d_backprop_filter(op.inputs[0], shape_1, grad, dilations=dilations, strides=strides, padding=padding, use_cudnn_on_gpu=use_cudnn_on_gpu, data_format=data_format) ]