def testIndexedSlicesGradientInCondInWhileLoop(self):
with ops.Graph().as_default():
embedding_matrix = tf.get_variable(
"embedding_matrix", [5, 5],
initializer=tf.random_normal_initializer())
def Cond(it, _):
return it < 5
def Body(it, cost):
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
cost = tf.cond(tf.equal(it, 3),
lambda: tf.square(cost),
lambda: cost + tf.reduce_sum(embedding))
return it + 1, cost
_, cost = control_flow_ops.While(
Cond, Body, [tf.constant(0), tf.constant(0.0)])
dynamic_grads = tf.gradients(cost, [embedding_matrix])[0]
dynamic_grads = tf.segment_sum(dynamic_grads.values,
dynamic_grads.indices)
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
static = tf.square(
tf.reduce_sum(embedding) +
tf.reduce_sum(embedding) +
tf.reduce_sum(embedding)) + tf.reduce_sum(embedding)
static_grads = tf.gradients(static, [embedding_matrix])[0]
static_grads = tf.segment_sum(static_grads.values, static_grads.indices)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
def _apply_activation_with_summaries(x, activation_fn):
"""Returns activation_fn(x).
This applies the given activation and adds useful summaries specific to the
activation.
Args:
x: The tensor to apply activation to.
activation_fn: An activation function.
Returns:
A tensor with activation applied to x.
"""
if activation_fn is None:
return x
y = activation_fn(x)
if activation_fn in (nn.relu, nn.softplus, nn.relu6):
# Using x for comparison to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.to_float(standard_ops.less(
x, 0.0))), '%s/zeros' % y.op.name)
if activation_fn is nn.relu6:
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.to_float(standard_ops.greater(
x, 6.0))), '%s/sixes' % y.op.name)
if activation_fn is nn.l2_normalize:
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.sum(
standard_ops.square(x), 1))), '%s/length' % y.op.name)
_add_histogram_summary(y, '%s/activations' % y.op.name)
return y
def _apply_activation_with_summaries(x, activation_fn):
"""Returns activation_fn(x).
This applies the given activation and adds useful summaries specific to the
activation.
Args:
x: The tensor to apply activation to.
activation_fn: An activation function.
Returns:
A tensor with activation applied to x.
"""
if activation_fn is None:
return x
y = activation_fn(x)
if activation_fn in (nn.relu, nn.softplus, nn.relu6):
# Using x for comparison to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(standard_ops.less(x, 0.0))),
'%s/zeros' % y.op.name)
if activation_fn is nn.relu6:
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(standard_ops.greater(x, 6.0))),
'%s/sixes' % y.op.name)
if activation_fn is nn.l2_normalize:
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.sqrt(standard_ops.sum(standard_ops.square(x),
1))),
'%s/length' % y.op.name)
_add_histogram_summary(y, '%s/activations' % y.op.name)
return y
def _apply_variational_kernel(self, inputs):
if (not isinstance(self.kernel_posterior, independent_lib.Independent)
or not isinstance(self.kernel_posterior.distribution,
normal_lib.Normal)):
raise TypeError(
"`DenseLocalReparameterization` requires "
"`kernel_posterior_fn` produce an instance of "
"`tf.distributions.Independent(tf.distributions.Normal)` "
"(saw: \"{}\").".format(self.kernel_posterior.name))
self.kernel_posterior_affine = normal_lib.Normal(
loc=self._matmul(inputs, self.kernel_posterior.distribution.loc),
scale=standard_ops.sqrt(
self._matmul(
standard_ops.square(inputs),
standard_ops.square(
self.kernel_posterior.distribution.scale))))
self.kernel_posterior_affine_tensor = (self.kernel_posterior_tensor_fn(
self.kernel_posterior_affine))
self.kernel_posterior_tensor = None
return self.kernel_posterior_affine_tensor
def _apply_variational_kernel(self, inputs):
if not self.kernel_use_local_reparameterization:
self.kernel.posterior_tensor = self.kernel.posterior_tensor_fn(
self.kernel.posterior)
self.kernel.posterior_affine = None
self.kernel.posterior_affine_tensor = None
return self._matmul(inputs, self.kernel.posterior_tensor)
if not isinstance(self.kernel.posterior, normal_lib.Normal):
raise TypeError("`kernel_use_local_reparameterization=True` requires "
"`kernel_posterior_fn` produce an instance of "
"`tf.distributions.Normal` (saw: \"{}\").".format(
type(self.kernel.posterior).__name__))
self.kernel.posterior_affine = normal_lib.Normal(
loc=self._matmul(inputs, self.kernel.posterior.loc),
scale=standard_ops.sqrt(self._matmul(
standard_ops.square(inputs),
standard_ops.square(self.kernel.posterior.scale))))
self.kernel.posterior_affine_tensor = (
self.kernel.posterior_tensor_fn(self.kernel.posterior_affine))
self.kernel.posterior_tensor = None
return self.kernel.posterior_affine_tensor
def _apply_variational_kernel(self, inputs):
if not self.kernel_use_local_reparameterization:
self.kernel.posterior_tensor = self.kernel.posterior_tensor_fn(
self.kernel.posterior)
self.kernel.posterior_affine = None
self.kernel.posterior_affine_tensor = None
return self._matmul(inputs, self.kernel.posterior_tensor)
if not isinstance(self.kernel.posterior, normal_lib.Normal):
raise TypeError(
"`kernel_use_local_reparameterization=True` requires "
"`kernel_posterior_fn` produce an instance of "
"`tf.distributions.Normal` (saw: \"{}\").".format(
type(self.kernel.posterior).__name__))
self.kernel.posterior_affine = normal_lib.Normal(
loc=self._matmul(inputs, self.kernel.posterior.loc),
scale=standard_ops.sqrt(
self._matmul(standard_ops.square(inputs),
standard_ops.square(
self.kernel.posterior.scale))))
self.kernel.posterior_affine_tensor = (self.kernel.posterior_tensor_fn(
self.kernel.posterior_affine))
self.kernel.posterior_tensor = None
return self.kernel.posterior_affine_tensor
def Body(it, cost):
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
cost = tf.cond(tf.equal(it, 3),
lambda: tf.square(cost),
lambda: cost + tf.reduce_sum(embedding))
return it + 1, cost
