def _fully_connected_grad(op, grad):
act = op.get_attr('act')
y = op.outputs[0]
if act == 'RELU':
grad = gen_nn_ops.relu_grad(grad, y)
elif act == 'RELU6':
grad = gen_nn_ops.relu6_grad(grad, y)
elif act == 'TANH':
y = math_ops.conj(y)
grad = gen_math_ops.tanh_grad(y, grad)
broadcast_shape = tf.shape(y)
input_value_shape = tf.shape(op.inputs[2])
_, reduction_axes = tf.raw_ops.BroadcastGradientArgs(s0=broadcast_shape,
s1=input_value_shape)
updates_grad_reshaped = tf.reduce_sum(grad,
axis=reduction_axes,
keepdims=True)
bias_grad = tf.reshape(updates_grad_reshaped, input_value_shape)
a = math_ops.conj(op.inputs[0])
b = math_ops.conj(op.inputs[1])
grad_a = gen_math_ops.mat_mul(grad, b)
grad_b = gen_math_ops.mat_mul(grad, a, transpose_a=True)
return [grad_a, grad_b, bias_grad]
def test_tanhgrad_3d(self):
y = constant_op.constant(
self.generate_random_numbers(60, 5.0, 30.0), shape=[10, 3, 2])
y_delta = constant_op.constant(
self.generate_random_numbers(60, 10.0, 40.0), shape=[10, 3, 2])
out = tanh_grad(y, y_delta)
def run_test(sess):
return sess.run(out)
assert np.allclose(
self.with_ngraph(run_test), self.without_ngraph(run_test))
def _conv_add_relu_grad(op, grad):
act = op.get_attr('act')
y = op.outputs[0]
if act == 'RELU':
grad = gen_nn_ops.relu_grad(grad, y)
elif act == 'RELU6':
grad = gen_nn_ops.relu6_grad(grad, y)
elif act == 'TANH':
y = math_ops.conj(y)
grad = gen_math_ops.tanh_grad(y, grad)
broadcast_shape = tf.shape(y)
input_value_shape = tf.shape(op.inputs[2])
_, reduction_axes = tf.raw_ops.BroadcastGradientArgs(
s0=broadcast_shape, s1=input_value_shape)
updates_grad_reshaped = tf.reduce_sum(
grad, axis=reduction_axes, keepdims=True)
bias_grad = tf.reshape(updates_grad_reshaped, input_value_shape)
dilations = [1, op.get_attr('dilation_w'), op.get_attr('dilation_h'), 1]
strides = [1, op.get_attr('stride_w'), op.get_attr('stride_h'), 1]
padding = op.get_attr('padding')
shape_0, shape_1 = tf.shape_n([op.inputs[0], op.inputs[1]])
return [
tf.compat.v1.nn.conv2d_backprop_input(
shape_0,
op.inputs[1],
grad,
strides=strides,
padding=padding,
dilations=dilations,
data_format='NHWC'),
tf.compat.v1.nn.conv2d_backprop_filter(
op.inputs[0],
shape_1,
grad,
strides=strides,
padding=padding,
dilations=dilations,
data_format='NHWC'), bias_grad
]
def _TanhGradGrad(op, grad):
with ops.control_dependencies([grad]):
a = math_ops.conj(op.inputs[0])
b = math_ops.conj(op.inputs[1])
return grad * -2.0 * b * a, gen_math_ops.tanh_grad(a, grad)
def _TanhGrad(op, grad):
"""Returns grad * (1 - tanh(x) * tanh(x))."""
y = op.outputs[0] # y = tanh(x)
with ops.control_dependencies([grad]):
y = math_ops.conj(y)
return gen_math_ops.tanh_grad(y, grad)
def _TanhGradGrad(op, grad):
with ops.control_dependencies([grad]):
a = math_ops.conj(op.inputs[0])
b = math_ops.conj(op.inputs[1])
return grad * -2.0 * b * a, gen_math_ops.tanh_grad(a, grad)
def _TanhGrad(op, grad):
"""Returns grad * (1 - tanh(x) * tanh(x))."""
y = op.outputs[0] # y = tanh(x)
with ops.control_dependencies([grad]):
y = math_ops.conj(y)
return gen_math_ops.tanh_grad(y, grad)
