def _Conv2DGrad(op, grad):
"""Weight sharing for symmetric lateral connections."""
strides = op.get_attr('strides')
padding = op.get_attr('padding')
use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu')
data_format = op.get_attr('data_format')
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
dx = nn_ops.conv2d_backprop_input(shape_0,
op.inputs[1],
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw = nn_ops.conv2d_backprop_filter(op.inputs[0],
shape_1,
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw_t = tf.transpose(dw, (2, 3, 0, 1))
dw_symm_t = (0.5) * (dw_t + tf.transpose(dw_t, (1, 0, 2, 3)))
dw_symm = tf.transpose(dw_symm_t, (2, 3, 0, 1))
return dx, dw_symm
def testConvBackpropFilter(self):
with self.session() as sess:
with ops.device("/device:IPU:0"):
inp = array_ops.placeholder(np.float32, [2, 8, 8, 3])
fil = constant_op.constant([2, 2, 3, 5], np.int32)
bck = array_ops.placeholder(np.float32, [2, 8, 8, 5],
name="wei")
output = nn_ops.conv2d_backprop_filter(inp,
fil,
bck,
strides=[1, 1, 1, 1],
padding="SAME")
report = tu.ReportJSON(self, sess)
report.reset()
fd = {
inp: np.zeros([2, 8, 8, 3]),
bck: np.zeros([2, 8, 8, 5]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros([2, 2, 3, 5]))
report.parse_log()
ok = [
'__seed*',
'Copy_',
'Conv2DBackpropFilter/convolution.*/Conv_8x8',
]
report.assert_all_compute_sets_and_list(ok)
def _Conv2DGrad(op, grad):
strides = op.get_attr("strides")
padding = op.get_attr("padding")
use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
data_format = op.get_attr("data_format")
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
dx = nn_ops.conv2d_backprop_input(
shape_0,
op.inputs[1],
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw = nn_ops.conv2d_backprop_filter(
op.inputs[0],
shape_1,
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw_t = tf.transpose(dw,(2,3,0,1))
dw_symm_t = (0.5) * (dw_t + tf.transpose(dw_t,(1,0,2,3)))
dw_symm = tf.transpose(dw_symm_t,(2,3,0,1))
return dx,dw_symm
def conv_scaled_inplace(input_values, grads, weights, lr):
return weights - lr * nn_ops.conv2d_backprop_filter(
input_values,
filter_sizes,
grads,
strides=[1, 1, 1, 1],
padding="SAME")
def testGradient(self):
with self.cached_session():
for padding in ["SAME", "VALID"]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [5, 8, 6, 4]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1,
dtype=dtypes.float32)
filter_shape = [3, 3, 4, 6]
# Make a convolution op with the current settings, just to easily get
# the shape of the output.
conv_out = nn_ops.conv2d(in_val,
array_ops.zeros(filter_shape),
strides=[1, stride, stride, 1],
padding=padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) - 1,
dtype=dtypes.float32)
output = nn_ops.conv2d_backprop_filter(
in_val,
filter_shape,
out_backprop_val,
strides=[1, stride, stride, 1],
padding=padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val],
[in_shape, out_backprop_shape], output, filter_shape)
print("conv2d_backprop_filter gradient err = %g " % err)
err_tolerance = 2e-3
self.assertLess(err, err_tolerance)
def _MpuSimConv2DGrad(op, grad):
"""Gradient function for MpuSimConv2D."""
dilations = op.get_attr("dilations")
strides = op.get_attr("strides")
padding = op.get_attr("padding")
data_format = op.get_attr("data_format")
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
# We call the gen_nn_ops backprop functions instead of nn_ops backprop
# functions for performance reasons in Eager mode. gen_nn_ops functions take a
# `explicit_paddings` parameter, but nn_ops functions do not. So if were were
# to use the nn_ops functions, we would have to convert `padding` and
# `explicit_paddings` into a single `padding` parameter, increasing overhead
# in Eager mode.
return [
nn_ops.conv2d_backprop_input(
shape_0,
op.inputs[1],
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=False,
data_format=data_format),
nn_ops.conv2d_backprop_filter(
op.inputs[0],
shape_1,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=False,
data_format=data_format)
]
def _Conv2DBackpropInputGrad(op, grad):
"""The derivatives for deconvolution.
Args:
op: the Deconvolution op.
grad: the tensor representing the gradient w.r.t. the output
Returns:
the gradients w.r.t. the input and the filter
"""
return [
None,
nn_ops.conv2d_backprop_filter(
grad,
array_ops.shape(op.inputs[1]),
op.inputs[2],
dilations=op.get_attr("dilations"),
strides=op.get_attr("strides"),
padding=op.get_attr("padding"),
use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
data_format=op.get_attr("data_format").decode()),
nn_ops.conv2d(
grad,
op.inputs[1],
dilations=op.get_attr("dilations"),
strides=op.get_attr("strides"),
padding=op.get_attr("padding"),
use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
data_format=op.get_attr("data_format").decode())
]
def _RunAndVerifyBackpropFilter(self, input_sizes, filter_sizes,
output_sizes, strides, padding, expected,
data_format, use_ve):
total_input_size = 1
total_output_size = 1
for s in input_sizes:
total_input_size *= s
for s in output_sizes:
total_output_size *= s
# Initializes the input tensor with array containing incrementing
# numbers from 1.
x0 = [f * 1.0 for f in range(1, total_input_size + 1)]
x2 = [f * 1.0 for f in range(1, total_output_size + 1)]
for dtype in self._DtypesToTest(use_ve=use_ve):
with test_util.device(use_ve=use_ve, use_gpu=False):
t0 = constant_op.constant(x0, shape=input_sizes, dtype=dtype)
t1 = constant_op.constant(filter_sizes,
shape=[len(filter_sizes)])
t2 = constant_op.constant(x2, shape=output_sizes, dtype=dtype)
explicit_strides = [1] + strides + [1]
if data_format == "NCHW":
t0 = test_util.NHWCToNCHW(t0)
t2 = test_util.NHWCToNCHW(t2)
explicit_strides = test_util.NHWCToNCHW(explicit_strides)
conv = nn_ops.conv2d_backprop_filter(t0,
t1,
t2,
strides=explicit_strides,
padding=padding,
data_format=data_format)
value = self.evaluate(conv)
self.assertShapeEqual(value, conv)
tf_logging.info("expected = ", expected)
tf_logging.info("actual = ", value)
self.assertArrayNear(expected, value.flatten(), 1e-5)
def tf_model(padding):
t1 = tf.compat.v1.placeholder(dtype=tf.float32,
shape=input_sizes_nhwc,
name='t1')
t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t2')
t3 = tf.compat.v1.placeholder(dtype=tf.float32,
shape=out_backprop_in_sizes[padding],
name='t3')
#reshaping the out_backprop to NHWC since TF does not support NCHW
t3 = tf.transpose(t3, [0, 2, 3, 1])
#Cast dtype to bfloat16 for TF because NNP casts ng_model inputs
t1 = tf.cast(t1, dtype=tf.bfloat16)
t3 = tf.cast(t3, dtype=tf.bfloat16)
filt = nn_ops.conv2d_backprop_filter(t1,
t2,
t3,
stride_nhwc,
padding=padding,
data_format='NHWC')
#Cast dtype back to float32 similar to NNP
filt = tf.cast(filt, dtype=tf.float32)
return filt, t1, t3
def _PerturbConv2DGrad(op, grad):
"""Set grads for the middle-most column to 0."""
strides = op.get_attr('strides')
padding = op.get_attr('padding')
use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu')
data_format = op.get_attr('data_format')
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
dx = nn_ops.conv2d_backprop_input(shape_0,
op.inputs[1],
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw = nn_ops.conv2d_backprop_filter(op.inputs[0],
shape_1,
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
# # Find middle unit
# h, w = shape_1[1] // 2, shape_1[2] // 2
# # Set middle unit gradient to 0
# dx[:, h, w] = 0.
# Find middle unit of hidden state (these are weights)
h, w = shape_0[1] // 2, shape_0[2] // 2
# Set middle unit gradient to 0
dw[:, h, w] = 0.
return dx, dw
def _Conv2DGrad(op, grad):
dilations = op.get_attr("dilations")
strides = op.get_attr("strides")
padding = op.get_attr("padding")
use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
data_format = op.get_attr("data_format")
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
return [
nn_ops.conv2d_backprop_input(
shape_0,
op.inputs[1],
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format),
nn_ops.conv2d_backprop_filter(
op.inputs[0],
shape_1,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
]
def _Conv2DBackpropInputGrad(op, grad):
"""The derivatives for deconvolution.
Args:
op: the Deconvolution op.
grad: the tensor representing the gradient w.r.t. the output
Returns:
the gradients w.r.t. the input and the filter
"""
return [
None,
nn_ops.conv2d_backprop_filter(
grad,
array_ops.shape(op.inputs[1]),
op.inputs[2],
dilations=op.get_attr("dilations"),
strides=op.get_attr("strides"),
padding=op.get_attr("padding"),
use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
data_format=op.get_attr("data_format")),
nn_ops.conv2d(grad,
op.inputs[1],
dilations=op.get_attr("dilations"),
strides=op.get_attr("strides"),
padding=op.get_attr("padding"),
use_cudnn_on_gpu=op.get_attr("use_cudnn_on_gpu"),
data_format=op.get_attr("data_format"))
]
def testGradientDilatedConv(self):
if test.is_gpu_available(cuda_only=True):
with self.test_session(use_gpu=True):
for padding in ["SAME", "VALID"]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [5, 8, 6, 4]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1, dtype=dtypes.float32)
filter_shape = [3, 3, 4, 6]
# Make a convolution op with the current settings,
# just to easily get the shape of the output.
conv_out = nn_ops.conv2d(
in_val,
array_ops.zeros(filter_shape),
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) - 1,
dtype=dtypes.float32)
output = nn_ops.conv2d_backprop_filter(
in_val,
filter_shape,
out_backprop_val,
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val], [in_shape, out_backprop_shape],
output, filter_shape)
print("conv2d_backprop_filter gradient err = %g " % err)
err_tolerance = 2e-3
self.assertLess(err, err_tolerance)
def _Conv2DGrad(op, grad):
dilations = op.get_attr("dilations")
strides = op.get_attr("strides")
padding = op.get_attr("padding")
use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
data_format = op.get_attr("data_format")
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
return [
nn_ops.conv2d_backprop_input(shape_0,
op.inputs[1],
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format),
nn_ops.conv2d_backprop_filter(op.inputs[0],
shape_1,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
]
def testConvBackpropFilter(self):
with ops.device("/device:IPU:0"):
inp = array_ops.placeholder(np.float32, [2, 8, 8, 3])
fil = constant_op.constant([2, 2, 3, 5], np.int32)
bck = array_ops.placeholder(np.float32, [2, 8, 8, 5], name="wei")
output = nn_ops.conv2d_backprop_filter(
inp, fil, bck, strides=[1, 1, 1, 1], padding="SAME")
with ops.device('cpu'):
report = gen_ipu_ops.ipu_event_trace()
tu.configure_ipu_system()
with tu.ipu_session() as sess:
sess.run(report)
fd = {
inp: np.zeros([2, 8, 8, 3]),
bck: np.zeros([2, 8, 8, 5]),
}
result = sess.run(output, fd)
self.assertAllClose(result, np.zeros([2, 2, 3, 5]))
result = sess.run(report)
s = tu.extract_all_strings_from_event_trace(result)
cs_list = tu.get_compute_sets_from_report(s)
ok = ['__seed*', 'Copy_', 'Conv2DBackpropFilter/convolution.*/Conv_8x8']
self.assertTrue(tu.check_all_compute_sets_and_list(cs_list, ok))
def _Conv2DGrad(op, grad):
return [
nn_ops.conv2d_backprop_input(
array_ops.shape(op.inputs[0]), op.inputs[1], grad, op.get_attr("strides"), op.get_attr("padding")
),
nn_ops.conv2d_backprop_filter(
op.inputs[0], array_ops.shape(op.inputs[1]), grad, op.get_attr("strides"), op.get_attr("padding")
),
]
def testBackwardFilterGradient(self):
np.random.seed(1)
in_shape = LayerShape(batch=8, height=128, width=128, channels=8)
filter_shape = FilterShape(height=3, width=3, in_channels=8, out_channels=8)
in_op = self._random_data_op(in_shape)
out_op = self._random_out_op(in_shape, filter_shape)
filter_gradient_op = nn_ops.conv2d_backprop_filter(
in_op, filter_shape, out_op, strides=_STRIDES, padding=_PADDING)
self._assert_reproducible(filter_gradient_op)
def _Conv2DGrad(op, grad):
return [nn_ops.conv2d_backprop_input(
array_ops.shape(op.inputs[0]), op.inputs[1], grad, op.get_attr("strides"),
op.get_attr("padding"), op.get_attr("use_cudnn_on_gpu"),
op.get_attr("data_format")),
nn_ops.conv2d_backprop_filter(op.inputs[0],
array_ops.shape(op.inputs[1]), grad,
op.get_attr("strides"),
op.get_attr("padding"),
op.get_attr("use_cudnn_on_gpu"),
op.get_attr("data_format"))]
def testBackwardFilterGradient(self):
in_shape = LayerShapeNHWC(batch=8, height=128, width=128, channels=8)
filter_shape = FilterShape2D(
height=3, width=3, in_channels=8, out_channels=8)
in_op = self._random_data_op(in_shape)
strides = [1, 1, 1, 1]
padding = 'SAME'
out_op = self._random_out_op(in_shape, filter_shape, strides, padding)
filter_gradient_op = nn_ops.conv2d_backprop_filter(
in_op, filter_shape, out_op, strides=strides, padding=padding)
self._assert_reproducible(filter_gradient_op)
def conv_backprop_filter_tf(inputmats,
filter_sizes,
out_backprop,
stride,
padding='VALID'):
strd = [1, stride, stride, 1]
d_w_backprop_filter = nn_ops.conv2d_backprop_filter(
input=inputmats,
filter_sizes=filter_sizes,
out_backprop=out_backprop,
strides=strd,
padding=padding)
return np.array(d_w_backprop_filter)
def ng_model(padding):
t1 = tf.placeholder(dtype=tf.float32, shape=input_sizes_nhwc, name='t1')
t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t1')
t3 = tf.placeholder(dtype=tf.float32,
shape=out_backprop_in_sizes[padding],
name='t3')
filt = nn_ops.conv2d_backprop_filter(t1,
t2,
t3,
stride,
padding=padding,
data_format='NHWC')
return filt, t1, t3
def testConvBackwardFilterGradient(self, rate=1):
in_shape = LayerShapeNHWC(batch=8, height=64, width=64, channels=8)
filter_shape = FilterShape2D(
height=3, width=3, in_channels=8, out_channels=8)
in_op = self._random_data_op(in_shape)
strides = [1, 1, 1, 1]
padding = 'SAME'
dilations = [1, rate, rate, 1]
out_op = self._random_out_op(in_shape, filter_shape, strides, padding,
dilations)
self._assert_reproducible(lambda: nn_ops.conv2d_backprop_filter(
in_op,
filter_shape,
out_op,
strides=strides,
padding=padding,
dilations=dilations))
def _DeConv2DGrad(op, grad):
"""The derivatives for deconvolution.
Args:
op: the Deconvolution op.
grad: the tensor representing the gradient w.r.t. the output
Returns:
the gradients w.r.t. the input and the filter
"""
return [
None,
nn_ops.conv2d_backprop_filter(
grad, array_ops.shape(op.inputs[1]), op.inputs[2], op.get_attr("strides"), op.get_attr("padding")
),
nn_ops.conv2d(grad, op.inputs[1], op.get_attr("strides"), op.get_attr("padding")),
]
def _DeConv2DGrad(op, grad):
"""The derivatives for deconvolution.
Args:
op: the Deconvolution op.
grad: the tensor representing the gradient w.r.t. the output
Returns:
the gradients w.r.t. the input and the filter
"""
return [
None,
nn_ops.conv2d_backprop_filter(grad, array_ops.shape(op.inputs[1]),
op.inputs[2], op.get_attr("strides"),
op.get_attr("padding")),
nn_ops.conv2d(grad, op.inputs[1], op.get_attr("strides"),
op.get_attr("padding"))
]
def testGradientDilatedConv(self):
if test.is_gpu_available(cuda_only=True):
with self.session():
for padding in [
"SAME", "VALID", [(0, 0), (3, 5), (2, 1), (0, 0)],
[(0, 0), (5, 2), (5, 1), (0, 0)]
]:
for stride in [1, 2]:
np.random.seed(1)
in_shape = [5, 8, 6, 4]
in_val = constant_op.constant(
2 * np.random.random_sample(in_shape) - 1,
dtype=dtypes.float32)
filter_shape = [3, 3, 4, 6]
# Make a convolution op with the current settings,
# just to easily get the shape of the output.
conv_out = nn_ops.conv2d(
in_val,
array_ops.zeros(filter_shape),
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
out_backprop_shape = conv_out.get_shape().as_list()
out_backprop_val = constant_op.constant(
2 * np.random.random_sample(out_backprop_shape) -
1,
dtype=dtypes.float32)
output = nn_ops.conv2d_backprop_filter(
in_val,
filter_shape,
out_backprop_val,
dilations=[1, 2, 2, 1],
strides=[1, stride, stride, 1],
padding=padding)
err = gradient_checker.compute_gradient_error(
[in_val, out_backprop_val],
[in_shape, out_backprop_shape], output,
filter_shape)
print("conv2d_backprop_filter gradient err = %g " %
err)
err_tolerance = 1e-2
self.assertLess(err, err_tolerance)
def _Conv2DGrad(op, grad):
return [
nn_ops.conv2d_backprop_input(
array_ops.shape(op.inputs[0]),
op.inputs[1],
grad,
op.get_attr("strides"),
op.get_attr("padding"),
op.get_attr("use_cudnn_on_gpu"),
op.get_attr("data_format"),
),
nn_ops.conv2d_backprop_filter(
op.inputs[0],
array_ops.shape(op.inputs[1]),
grad,
op.get_attr("strides"),
op.get_attr("padding"),
op.get_attr("use_cudnn_on_gpu"),
op.get_attr("data_format"),
),
]
def tf_model(padding):
t1 = tf.placeholder(dtype=tf.float32, shape=input_sizes_nhwc, name='t1')
t2 = tf.constant(filter_size_hwio, dtype=tf.int32, name='t1')
t3 = tf.placeholder(dtype=tf.float32,
shape=out_backprop_in_sizes[padding],
name='t3')
#Cast dtype to bfloat16 for TF because NNP casts ng_model inputs
t1 = tf.cast(t1, dtype=tf.bfloat16)
t3 = tf.cast(t3, dtype=tf.bfloat16)
filt = nn_ops.conv2d_backprop_filter(t1,
t2,
t3,
stride,
padding=padding,
data_format='NHWC')
#Cast dtype back to float32 similar to NNP
filt = tf.cast(filt, dtype=tf.float32)
return filt, t1, t3
def _SConv2DGrad(op, grad):
"""Weight sharing for symmetric lateral connections."""
strides = op.get_attr('strides')
padding = op.get_attr('padding')
use_cudnn_on_gpu = op.get_attr('use_cudnn_on_gpu')
data_format = op.get_attr('data_format')
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
dx = nn_ops.conv2d_backprop_input(shape_0,
op.inputs[1],
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw = nn_ops.conv2d_backprop_filter(op.inputs[0],
shape_1,
grad,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
dw = 0.5 * (dw + dw[::-1, ::-1, :, :])
return dx, dw
def test_conv2d_grad_weight():
''' Run tests on the Wave custom conv2d operator.
'''
tf.reset_default_graph()
# Turn off graph-rewriting optimizations
config = tf.ConfigProto(graph_options=tf.GraphOptions(
optimizer_options=tf.OptimizerOptions(
opt_level=tf.OptimizerOptions.L0)))
# NCHW
t_n = 1
t_ci = 2
t_co = 4
t_h = 4
t_w = t_h
w_k = 3
t_ho = t_h - (w_k - 1)
t_wo = t_w - (w_k - 1)
if True:
# N H W C
gradient_same = tf.get_variable(
"gs", [t_n, t_h, t_w, t_co],
dtype=tf.float32,
initializer=tf.glorot_normal_initializer())
gradient_valid = tf.get_variable(
"gv", [t_n, t_ho, t_wo, t_co],
dtype=tf.float32,
initializer=tf.glorot_normal_initializer())
# K K Ci Co
c2d_in = tf.get_variable("f", [t_n, t_h, t_w, t_ci],
dtype=tf.float32,
initializer=tf.glorot_normal_initializer())
total_iters = 100
else:
total_iters = 100
'''
# N H W C
gradient_same = tf.get_variable("gs", dtype=tf.float32,
initializer=[[[[ 0.20130304, 0.27111197, -0.23086302, -0.23035139],
[-0.09796654, -0.25246394, -0.37301826, -0.04510078],
[ 0.23488244, 0.11107023, -0.06509414, -0.10155579],
[-0.02653619, 0.25628388, 0.14519225, -0.03400368]],
[[-0.2737653, -0.09805538, 0.2395286, -0.05114909],
[ 0.25615498, -0.00413944, -0.11665037, 0.41589662],
[-0.2739116, -0.25920913, -0.09962016, 0.09206288],
[ 0.20151283, 0.17594334, -0.07801611, 0.18326335]],
[[ 0.41815385, -0.2302951, 0.04820603, 0.28555122],
[-0.07512673, 0.03257893, 0.26206508, -0.23212446],
[ 0.11086888, -0.14072362, 0.0384812, -0.3119425 ],
[ 0.12364136, 0.20889276, 0.21803159, 0.23013732]],
[[ 0.14886299, 0.2521119, 0.09665762, -0.0950445 ],
[-0.02764839, -0.22908632, -0.21170329, -0.01173863],
[-0.01734301, -0.09034941, -0.36093128, -0.10678421],
[ 0.22199751, -0.07065484, -0.20354767, 0.35294214]]]])
# K K Ci Co
c2d_in = tf.get_variable("f", dtype=tf.float32,
initializer=[[[[ 0.05964299, -0.05988665],
[ 0.1176498, -0.03819368],
[ 0.5281046, -0.23744191],
[ 0.11645658, -0.19929294]],
[[-0.23003536, 0.00503353],
[-0.13039683, 0.07515113],
[ 0.27454942, 0.07771762],
[-0.30660862, 0.0843183 ]],
[[ 0.45593834, -0.10620885],
[-0.2692576, -0.21969746],
[-0.0423032, -0.27153978],
[-0.12364867, 0.4992814 ]],
[[ 0.199746, -0.36852396],
[ 0.06857206, -0.5132366 ],
[-0.3468758, -0.11179192],
[-0.00362369, 0.24938436]]]])
'''
# N H W C
gradient_valid = tf.get_variable(
"gv",
dtype=tf.float32,
initializer=[[[[-0.46859148, -0.47966853, 0.20002109, -0.56815886],
[0.28490868, -0.34714422, -0.02198983,
-0.16817461]],
[[0.55926424, 0.43453974, -0.14650254, -0.35478827],
[0.33911255, -0.13839267, 0.50347936,
-0.24987353]]]])
# K K Ci Co
c2d_in = tf.get_variable("f",
dtype=tf.float32,
initializer=[[[[-0.02360273, 0.11748651],
[-0.06789866, -0.09688393],
[0.19889985, -0.06614035],
[0.1964069, -0.49319836]],
[[-0.11062168, 0.10862893],
[-0.11725109, -0.25628847],
[-0.03851736, -0.04931202],
[0.21630755, -0.22763665]],
[[0.22559226, 0.38029733],
[0.10525792, -0.36071476],
[0.05696308, -0.12789722],
[0.07823437, 0.0835037]],
[[0.2075723, 0.14272317],
[0.50318813, 0.3311627],
[-0.0821431, 0.15861501],
[-0.0524208, -0.01618423]]]])
wgt_shape = tf.get_variable("s",
dtype=tf.int32,
initializer=[w_k, w_k, t_ci, t_co])
t_init = tf.global_variables_initializer()
for i in range(total_iters):
# SAME variant
with tf.Session('', config=config) as sess:
t_init.run()
# print("Wave Kernel:\n-------------------------------------------------")
#z_op = nn_ops.conv2d_backprop_filter(
z_op = waveflow.wavecomp_ops_module.wave_conv2d_gradient_weight(
c2d_in,
wgt_shape,
gradient_same,
strides=[1, 1, 1, 1],
padding='SAME',
use_cudnn_on_gpu=False,
data_format='NHWC')
# Base tensorflow. Only supports NHWC.
z2_op = nn_ops.conv2d_backprop_filter(
#z2_op = wavecomp_ops_module.wave_conv2d_gradient_weight(
c2d_in,
wgt_shape,
gradient_same,
strides=[1, 1, 1, 1],
padding='SAME',
use_cudnn_on_gpu=False,
data_format='NHWC')
# z = z_op.eval()
# z2 = z2_op.eval()
z, z2, grad_val, in_val = sess.run(
[z_op, z2_op, gradient_same, c2d_in])
# print("\nTF:\n-------------------------------------------------")
assert_str = "Failure on i: %d, mode: SAME" % (i)
if not compare_tensor(z, z2, assert_str):
print("gradients: %s" % (grad_val))
print("c2d_in: %s" % (in_val))
print("z: shape: %s, %s" % (z.shape, z))
print("z (np): shape: %s, %s" % (z2.shape, z2))
print("\n\n")
assert False
# Valid variant
with tf.Session('', config=config) as sess:
t_init.run()
# print("Wave Kernel:\n-------------------------------------------------")
#z_op = nn_ops.conv2d_backprop_filter(
z_op = waveflow.wavecomp_ops_module.wave_conv2d_gradient_weight(
c2d_in,
wgt_shape,
gradient_valid,
strides=[1, 1, 1, 1],
padding='VALID',
use_cudnn_on_gpu=False,
data_format='NHWC')
# Base tensorflow. Only supports NHWC.
z2_op = nn_ops.conv2d_backprop_filter(
#z2_op = wavecomp_ops_module.wave_conv2d_gradient_weight(
c2d_in,
wgt_shape,
gradient_valid,
strides=[1, 1, 1, 1],
padding='VALID',
use_cudnn_on_gpu=False,
data_format='NHWC')
z, z2, grad_val, in_val = sess.run(
[z_op, z2_op, gradient_valid, c2d_in])
# print("\nTF:\n-------------------------------------------------")
assert_str = "Failure on i: %d, mode: VALID" % (i)
if not compare_tensor(z, z2, assert_str):
print("gradients: %s" % (grad_val))
print("c2d_in: %s" % (in_val))
print("z: shape: %s, %s" % (z.shape, z))
print("z (np): shape: %s, %s" % (z2.shape, z2))
print("\n\n")
assert False
return True
def _Conv2DGrad(op, grad):
"""Gradient function for Conv2D."""
dilations = op.get_attr("dilations")
strides = op.get_attr("strides")
padding = op.get_attr("padding")
explicit_paddings = op.get_attr("explicit_paddings")
use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
data_format = op.get_attr("data_format")
use_cudnn_on_gpu = op.get_attr("use_cudnn_on_gpu")
shape_0, shape_1 = array_ops.shape_n([op.inputs[0], op.inputs[1]])
enable_quantop_grad = int(os.getenv('ENABLE_QUANTOP_CONV_GRAD', 0))
enable_quantop_input = int(os.getenv('ENABLE_QUANTOP_CONV', 0))
enable_quantop_wtgrad = int(os.getenv('ENABLE_QUANTOP_CONV_WTGRAD', 0))
dformat = 'channels_last'
inp_channels = op.inputs[0].get_shape()[3].value
if data_format == b'NCHW':
dformat = 'channels_first'
inp_channels = op.inputs[0].get_shape()[1].value
elif data_format == b'None':
dformat = 'unknown'
quant_input_copy = int(os.getenv('QUANTEMU_ALLOCATE_COPY_INPUTS', 23))
quant_filter_copy = int(os.getenv('QUANTEMU_ALLOCATE_COPY_FILTERS', 23))
quant_input_precision = int(os.getenv('QUANTEMU_PRECISION_CONV_INPUTS',
23))
quant_filter_precision = int(
os.getenv('QUANTEMU_PRECISION_CONV_FILTERS', 23))
quant_grad_precision = int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23))
quant_wtgrad_precision = int(
os.getenv('QUANTEMU_PRECISION_CONV_WTGRADS', 23))
if inp_channels == 3:
quant_grad_precision = quant_input_precision = int(
os.getenv('QUANTEMU_FIRST_LAYER_PRECISION', 23))
quant_filter_precision = int(
os.getenv('QUANTEMU_FIRST_LAYER_PRECISION', 23))
if enable_quantop_grad == 1:
grad = quantemu_ops.quantize_emu(
grad,
data_format=dformat,
data_type=int(os.getenv('QUANTEMU_GRAD_DATA_TYPE', 0)),
precision=
quant_grad_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23)),
exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)),
channel_blocking_type=int(
os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_GRADS', 0)),
channels_per_block=int(os.getenv('QUANTEMU_CBLOCK_SIZE_GRADS', 0)),
round_mode=int(os.getenv('QUANTEMU_RMODE_GRADS', 0)))
if enable_quantop_input == 1:
if quant_input_copy == 1:
acts = quantemu_ops.quantize_emu(
op.inputs[0],
data_format=dformat,
data_type=int(os.getenv('QUANTEMU_INPUT_DATA_TYPE', 0)),
precision=
quant_input_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_INPUTS', 23)),
exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)),
channel_blocking_type=int(
os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_INPUTS', 0)),
channels_per_block=int(
os.getenv('QUANTEMU_CBLOCK_SIZE_INPUTS', 0)),
round_mode=int(os.getenv('QUANTEMU_RMODE_INPUTS', 0)))
else:
acts = op.inputs[0]
if quant_filter_copy == 1:
filters = quantemu_ops.quantize_emu(
op.inputs[1],
data_format=dformat,
allocate_copy=int(
os.getenv('QUANTEMU_ALLOCATE_COPY_FILTERS', 0)),
data_type=int(os.getenv('QUANTEMU_FILTER_DATA_TYPE', 0)),
precision=
quant_filter_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_FILTERS', 23)),
exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)),
channel_blocking_type=int(
os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_FILTERS', 0)),
channels_per_block=int(
os.getenv('QUANTEMU_CBLOCK_SIZE_FILTERS', 0)),
round_mode=int(os.getenv('QUANTEMU_RMODE_FILTERS', 0)))
else:
filters = op.inputs[1]
outgrad = nn_ops.conv2d_backprop_input(
shape_0,
#op.inputs[1],
filters,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
wtgrad = nn_ops.conv2d_backprop_filter(
#op.inputs[0],
acts,
shape_1,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
if enable_quantop_wtgrad == 1:
wtgrad = quantemu_ops.quantize_emu(
wtgrad,
data_format=dformat,
data_type=int(os.getenv('QUANTEMU_WTGRAD_DATA_TYPE', 0)),
precision=
quant_wtgrad_precision, #int(os.getenv('QUANTEMU_PRECISION_CONV_GRADS', 23)),
exponent_bits=int(os.getenv('QUANTEMU_EXPBITS', 5)),
channel_blocking_type=int(
os.getenv('QUANTEMU_CBLOCK_TYPE_CONV_WTGRADS', 0)),
channels_per_block=int(
os.getenv('QUANTEMU_CBLOCK_SIZE_WTGRADS', 0)),
round_mode=int(os.getenv('QUANTEMU_RMODE_WTGRADS', 0)))
return [outgrad, wtgrad]
else: # No Quantization
return [
nn_ops.conv2d_backprop_input(shape_0,
op.inputs[1],
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format),
nn_ops.conv2d_backprop_filter(op.inputs[0],
shape_1,
grad,
dilations=dilations,
strides=strides,
padding=padding,
use_cudnn_on_gpu=use_cudnn_on_gpu,
data_format=data_format)
]
