def _BiasAddGrad(op, received_grad):
"""Return the gradients for the 2 inputs of bias_op.
The first input of unused_bias_op is the tensor t, and its gradient is
just the gradient the unused_bias_op received.
The second input of unused_bias_op is the bias vector which has one fewer
dimension than "received_grad" (the batch dimension.) Its gradient is the
received gradient Summed on the batch dimension, which is the first dimension.
Args:
op: The BiasOp for which we need to generate gradients.
received_grad: Tensor. The gradients passed to the BiasOp.
Returns:
Two tensors, the first one for the "tensor" input of the BiasOp,
the second one for the "bias" input of the BiasOp.
"""
try:
data_format = op.get_attr("data_format")
except ValueError:
data_format = None
return (received_grad,
gen_nn_ops.bias_add_grad(out_backprop=received_grad,
data_format=data_format))
def _BiasAddGrad(op, received_grad):
"""Return the gradients for the 2 inputs of bias_op.
The first input of unused_bias_op is the tensor t, and its gradient is
just the gradient the unused_bias_op received.
The second input of unused_bias_op is the bias vector which has one fewer
dimension than "received_grad" (the batch dimension.) Its gradient is the
received gradient Summed on the batch dimension, which is the first dimension.
Args:
op: The BiasOp for which we need to generate gradients.
received_grad: Tensor. The gradients passed to the BiasOp.
Returns:
Two tensors, the first one for the "tensor" input of the BiasOp,
the second one for the "bias" input of the BiasOp.
"""
try:
data_format = op.get_attr("data_format")
except ValueError:
data_format = None
return (received_grad,
gen_nn_ops.bias_add_grad(
out_backprop=received_grad, data_format=data_format))
def test_vecadd_grad():
''' Run tests on the Wave custom vector add operator.
'''
tf.reset_default_graph()
v_a1 = tf.get_variable("a1", [10, 5], dtype=tf.float32, initializer=tf.glorot_normal_initializer())
v_a2 = tf.get_variable("a2", [3, 2, 10, 6], dtype=tf.float32, initializer=tf.glorot_normal_initializer())
t_init = tf.global_variables_initializer()
t_debug = False
for i in range(100):
with tf.Session('') as sess:
t_init.run()
if t_debug:
print("Wave Kernel:\n-------------------------------------------------")
print("dims: a: %s" % (v_a1.shape))
z_op = waveflow.wavecomp_ops_module.wave_vec_add_grad(v_a1)
z_tf_op = gen_nn_ops.bias_add_grad(v_a1)
z, z_tf, a1 = sess.run([z_op, z_tf_op, v_a1])
if t_debug:
print("a: %s" % (a1))
print("z: %s" % (z))
print("z (tf): %s" % (z_tf))
assert np.allclose(z, z_tf)
with tf.Session('') as sess:
t_init.run()
if t_debug:
print("Wave Kernel:\n-------------------------------------------------")
print("dims: a: %s" % (v_a2.shape))
z2_op = waveflow.wavecomp_ops_module.wave_vec_add_grad(v_a2)
z2_tf_op = gen_nn_ops.bias_add_grad(v_a2)
z2, z2_tf, a2 = sess.run([z2_op, z2_tf_op, v_a2])
if t_debug:
print("a: %s" % (a2))
print("z: %s" % (z2))
print("z (tf): %s" % (z2_tf))
assert np.allclose(z2, z2_tf)
return True
def _LSTMBlockCellGrad(op, *grad): # pylint:disable=invalid-name,g-wrong-blank-lines
"""Gradient for LSTMBlockCell."""
(x, cs_prev, h_prev, w, wci, wcf, wco, b) = op.inputs
(i, cs, f, o, ci, co, _) = op.outputs
(_, cs_grad, _, _, _, _, h_grad) = grad
batch_size = x.get_shape().with_rank(2).dims[0].value
if batch_size is None:
batch_size = -1
input_size = x.get_shape().with_rank(2).dims[1].value
if input_size is None:
raise ValueError("input_size from `x` should not be None.")
cell_size = cs_prev.get_shape().with_rank(2).dims[1].value
if cell_size is None:
raise ValueError("cell_size from `cs_prev` should not be None.")
(cs_prev_grad, dgates, wci_grad, wcf_grad,
wco_grad) = tf.raw_ops.LSTMBlockCellGrad(
x=x,
cs_prev=cs_prev,
h_prev=h_prev,
w=w,
wci=wci,
wcf=wcf,
wco=wco,
b=b,
i=i,
cs=cs,
f=f,
o=o,
ci=ci,
co=co,
cs_grad=cs_grad,
h_grad=h_grad,
use_peephole=op.get_attr("use_peephole"))
# Backprop from dgates to xh.
xh_grad = tf.matmul(dgates, w, transpose_b=True)
x_grad = tf.slice(xh_grad, (0, 0), (batch_size, input_size))
x_grad.get_shape().merge_with(x.get_shape())
h_prev_grad = tf.slice(xh_grad, (0, input_size), (batch_size, cell_size))
h_prev_grad.get_shape().merge_with(h_prev.get_shape())
# Backprop from dgates to w.
xh = tf.concat([x, h_prev], 1)
w_grad = tf.matmul(xh, dgates, transpose_a=True)
w_grad.get_shape().merge_with(w.get_shape())
# Backprop from dgates to b.
b_grad = gen_nn_ops.bias_add_grad(dgates)
b_grad.get_shape().merge_with(b.get_shape())
return (x_grad, cs_prev_grad, h_prev_grad, w_grad, wci_grad, wcf_grad,
wco_grad, b_grad)
def testBiasAddGrad(self):
self._testUnary(gen_nn_ops.bias_add_grad,
np.array([[1., 2.], [3., 4.]], dtype=np.float32),
expected=np.array([4., 6.], dtype=np.float32))
self._testUnary(
lambda x: gen_nn_ops.bias_add_grad(x, data_format="NCHW"),
np.array([[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]],
dtype=np.float32),
expected=np.array([10., 26.], dtype=np.float32))
def testBiasAddGrad(self):
self._testUnary(
gen_nn_ops.bias_add_grad,
np.array([[1., 2.], [3., 4.]], dtype=np.float32),
expected=np.array([4., 6.], dtype=np.float32))
self._testUnary(lambda x: gen_nn_ops.bias_add_grad(x, data_format="NCHW"),
np.array([[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]],
dtype=np.float32),
expected=np.array([10., 26.], dtype=np.float32))
def testBiasAddGrad(self):
self._assertOpOutputMatchesExpected(
gen_nn_ops.bias_add_grad,
np.array([[1., 2.], [3., 4.]], dtype=np.float32),
expected=np.array([4., 6.], dtype=np.float32))
self._assertOpOutputMatchesExpected(
lambda x: gen_nn_ops.bias_add_grad(x, data_format="NCHW"),
np.array(
[[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]], dtype=np.float32),
expected=np.array([14., 22.], dtype=np.float32))
def testBiasAddGrad(self):
self._assertOpOutputMatchesExpected(
gen_nn_ops.bias_add_grad,
np.array([[1., 2.], [3., 4.]], dtype=np.float32),
expected=np.array([4., 6.], dtype=np.float32))
self._assertOpOutputMatchesExpected(
lambda x: gen_nn_ops.bias_add_grad(x, data_format="NCHW"),
np.array(
[[[1., 2.], [3., 4.]], [[5., 6.], [7., 8.]]], dtype=np.float32),
expected=np.array([14., 22.], dtype=np.float32))
def loop_fn(grad_y):
return gen_nn_ops.bias_add_grad(tf.expand_dims(grad_y, 0),
data_format=data_format)
def test_vecadd_grad_int():
''' Run tests on the Wave custom vector add operator.
'''
tf.reset_default_graph()
v_a1 = tf.get_variable("a1", [10, 5],
dtype=tf.float32,
initializer=tf.glorot_normal_initializer())
v_a2 = tf.get_variable("a2", [3, 201, 1001, 9],
dtype=tf.float32,
initializer=tf.glorot_normal_initializer())
t_init = tf.global_variables_initializer()
t_debug = False
for i in range(10):
with tf.Session('') as sess:
t_init.run()
if t_debug:
print(
"Wave Kernel:\n-------------------------------------------------"
)
print("dims: a: %s" % (v_a1.shape))
z_op = waveflow.wavecomp_ops_module.wave_vec_add_grad_int(v_a1)
z_tf_op = gen_nn_ops.bias_add_grad(v_a1)
z, z_tf, a1 = sess.run([z_op, z_tf_op, v_a1])
if t_debug:
print("a: %s" % (a1))
print("z: %s" % (z))
print("z (tf): %s" % (z_tf))
assert_str = "Failure on i: %d, params: %s" % (i, str(v_a1.shape))
if not compare_tensor(z, z_tf, assert_str):
print("a: %s" % (a1))
print("z: %s" % (z))
print("z (np): %s" % (z_tf))
print("\n\n")
assert False
with tf.Session('') as sess:
t_init.run()
if t_debug:
print(
"Wave Kernel:\n-------------------------------------------------"
)
print("dims: a: %s" % (v_a2.shape))
z2_op = waveflow.wavecomp_ops_module.wave_vec_add_grad_int(v_a2)
z2_tf_op = gen_nn_ops.bias_add_grad(v_a2)
z2, z2_tf, a2 = sess.run([z2_op, z2_tf_op, v_a2])
if t_debug:
print("a: %s" % (a2))
print("z: %s" % (z2))
print("z (tf): %s" % (z2_tf))
assert_str = "Failure on i: %d, params: %s" % (i, str(v_a2.shape))
if not compare_tensor(z2, z2_tf, assert_str):
print("a: %s" % (a2))
print("z: %s" % (z2))
print("z (np): %s" % (z2_tf))
print("\n\n")
assert False
return True