def testIndexedSlicesGradientInCondInWhileLoop(self):
with ops.Graph().as_default():
embedding_matrix = tf.get_variable(
"embedding_matrix", [5, 5],
initializer=tf.random_normal_initializer())
def Cond(it, _):
return it < 5
def Body(it, cost):
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
cost = tf.cond(tf.equal(it, 3),
lambda: tf.square(cost),
lambda: cost + tf.reduce_sum(embedding))
return it + 1, cost
_, cost = control_flow_ops.While(
Cond, Body, [tf.constant(0), tf.constant(0.0)])
dynamic_grads = tf.gradients(cost, [embedding_matrix])[0]
dynamic_grads = tf.segment_sum(dynamic_grads.values,
dynamic_grads.indices)
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
static = tf.square(
tf.reduce_sum(embedding) +
tf.reduce_sum(embedding) +
tf.reduce_sum(embedding)) + tf.reduce_sum(embedding)
static_grads = tf.gradients(static, [embedding_matrix])[0]
static_grads = tf.segment_sum(static_grads.values, static_grads.indices)
with self.test_session() as sess:
sess.run(tf.initialize_all_variables())
self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
def li(weights, name=None):
"""Applies li regularization to weights."""
with ops.op_scope([weights], name, 'li_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
name=scope)
def lo(weights, name=None):
"""Applies group column regularization to weights."""
with ops.op_scope([weights], name, 'lo_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
name=scope)
def while_loop_body(iteration, multipliers, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = standard_ops.minimum(
0.0, (radius - standard_ops.reduce_sum(multipliers)) /
standard_ops.maximum(1.0, standard_ops.reduce_sum(inactive)))
multipliers = multipliers + (scale * inactive)
new_inactive = standard_ops.cast(multipliers > 0, multipliers.dtype)
multipliers = multipliers * new_inactive
return (iteration, multipliers, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = (1.0 - standard_ops.reduce_sum(
matrix, axis=0, keepdims=True)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True))
matrix += scale * inactive
new_inactive = standard_ops.cast(matrix > 0, matrix.dtype)
matrix *= new_inactive
return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = (1.0 - standard_ops.reduce_sum(
matrix, axis=0, keepdims=True)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True))
matrix = matrix + (scale * inactive)
new_inactive = standard_ops.cast(matrix > 0, matrix.dtype)
matrix = matrix * new_inactive
return (iteration, matrix, new_inactive, inactive)
def lo(weights, name='lo_regularizer'):
"""Applies group column regularization to weights."""
with tf.name_scope(name) as scope:
my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
# if tf.__version__ <= '0.12':
# standard_ops_fn = standard_ops.mul
# else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(
my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
name=scope
)
def while_loop_body(iteration, multipliers, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = standard_ops.minimum(
0.0,
(radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive)))
multipliers += scale * inactive
new_inactive = standard_ops.cast(multipliers > 0, multipliers.dtype)
multipliers *= new_inactive
return (iteration, multipliers, new_inactive, inactive)
def group(weights):
"""Applies group regularization to weights."""
with tf.name_scope('group_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(my_scale,
standard_ops.reduce_sum(
standard_ops.sqrt(
standard_ops.reduce_sum(
tf.square(weights), 1))),
name=scope)
def lo(weights, name='lo_regularizer'):
"""Applies group column regularization to weights."""
with tf.name_scope(name) as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
if tf.__version__ <= '0.12':
standard_ops_fn = standard_ops.mul
else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
name=scope)
def li(weights, name=None):
"""Applies li regularization to weights."""
with tf.name_scope('li_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
# if tf.__version__ <= '0.12':
# standard_ops_fn = standard_ops.mul
# else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
name=scope)
def mn_i(weights, name=None):
"""Applies max-norm regularization to weights."""
try: # TF12
with ops.name_scope(scope, 'maxnorm_i_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
except: # TF11
with ops.op_scope([weights], name, 'maxnorm_i_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def li(weights, name=None):
"""Applies li regularization to weights."""
with tf.name_scope('li_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
if tf.__version__ <= '0.12':
standard_ops_fn = standard_ops.mul
else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
name=scope)
def lo(weights, name=None):
"""Applies group column regularization to weights."""
with ops.op_scope([weights], name, 'lo_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
# return standard_ops.mul(
# my_scale,
# standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(weights**2, 0))),
# name=scope)
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))),
# standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.reduce_mean(tf.square(weights), 0))),
name=scope)
def mn_i(weights, name=None):
"""Applies max-norm regularization to weights."""
with ops.op_scope([weights], name, 'maxnorm_o_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def testIndexedSlicesWithDynamicShapeGradientInWhileLoop(self):
for dtype in [dtypes.float32, dtypes.float64]:
with self.test_session() as sess:
inputs = tf.placeholder(dtype=dtype)
initial_outputs = tf.TensorArray(dtype=dtype,
dynamic_size=True,
size=1)
initial_i = tf.constant(0, dtype=dtypes.int32)
def Cond(i, _):
return i < tf.size(inputs) # pylint: disable=cell-var-from-loop
def Body(i, outputs):
x = tf.gather(inputs, i) # pylint: disable=cell-var-from-loop
outputs = outputs.write(i, x)
return i + 1, outputs
_, outputs = tf.while_loop(Cond, Body,
[initial_i, initial_outputs])
outputs = tf.reduce_sum(outputs.pack())
r = tf.gradients([outputs], [inputs])[0]
grad_wr_inputs = ops.convert_to_tensor(r)
o, grad = sess.run([outputs, grad_wr_inputs],
feed_dict={inputs: [1, 3, 2]})
self.assertEquals(o, 6)
self.assertAllEqual(grad, [1] * 3)
def testIndexedSlicesWithDynamicShapeGradientInWhileLoop(self):
for dtype in [dtypes.float32, dtypes.float64]:
with self.test_session() as sess:
inputs = tf.placeholder(dtype=dtype)
initial_outputs = tf.TensorArray(dtype=dtype, dynamic_size=True,
size=1)
initial_i = tf.constant(0, dtype=dtypes.int32)
def Cond(i, _):
return i < tf.size(inputs) # pylint: disable=cell-var-from-loop
def Body(i, outputs):
x = tf.gather(inputs, i) # pylint: disable=cell-var-from-loop
outputs = outputs.write(i, x)
return i + 1, outputs
_, outputs = tf.while_loop(Cond, Body, [initial_i, initial_outputs])
outputs = tf.reduce_sum(outputs.pack())
r = tf.gradients([outputs], [inputs])[0]
grad_wr_inputs = ops.convert_to_tensor(r)
o, grad = sess.run([outputs, grad_wr_inputs],
feed_dict={inputs: [1, 3, 2]})
self.assertEquals(o, 6)
self.assertAllEqual(grad, [1] * 3)
def testIndexedSlicesWithShapeGradientInWhileLoop(self):
with self.test_session() as sess:
num_steps = 9
inputs = tf.placeholder(dtype="float32", shape=[num_steps])
initial_outputs = tf.TensorArray(dtype="float32", size=num_steps)
initial_i = tf.constant(0, dtype="int32")
def Cond(i, _):
return i < num_steps
def Body(i, outputs):
x = tf.gather(inputs, i)
outputs = outputs.write(i, x)
return i + 1, outputs
_, outputs = tf.while_loop(Cond, Body, [initial_i, initial_outputs])
outputs = tf.reduce_sum(outputs.pack())
r = tf.gradients([outputs], [inputs])[0]
grad_wr_inputs = ops.convert_to_tensor(r)
o, grad = sess.run([outputs, grad_wr_inputs],
feed_dict={inputs: [4, 6, 0, 7, 0, 0, 1, 2, 0]})
self.assertEquals(o, 20)
self.assertAllEqual(grad, [1] * num_steps)
def mn_o(weights, name=None):
"""Applies max-norm regularization to weights."""
with ops.op_scope([weights], name, 'maxnorm_o_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 0)), name=scope)
def lp(weights):
"""Applies l2 regularization to weights."""
with ops.name_scope(scope, 'lp_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
reg_loss = standard_ops.reduce_sum(math_ops.pow(math_ops.abs(weights), p))
return standard_ops.multiply(my_scale, reg_loss, name=name)
def li(weights, name=None):
"""Applies li regularization to weights."""
with ops.op_scope([weights], name, 'li_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
# return standard_ops.mul(
# my_scale,
# standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(weights**2, 1))),
# name=scope)
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(
standard_ops.sqrt(
standard_ops.reduce_sum(tf.square(weights), 1))),
# standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.reduce_mean(tf.square(weights), 1))),
name=scope)
def mn_i(weights, name='maxnorm_i_regularizer'):
"""Applies max-norm regularization to weights."""
with tf.name_scope(name) as scope:
my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
if tf.__version__ <= '0.12':
standard_ops_fn = standard_ops.mul
else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def sum_reg(weights, name=None):
"""Applies sum regularization to weights."""
with ops.name_scope(scope, 'sum_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.multiply(my_scale,
standard_ops.reduce_sum(weights),
name=name)
def l1(weights, name=None):
"""Applies L1 regularization to weights."""
with ops.op_scope([weights], scope, 'l1_regularizer') as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(my_scale,
standard_ops.reduce_sum(
standard_ops.abs(weights)),
name=name)
def l1(weights, name=None):
"""Applies L1 regularization to weights."""
with ops.name_scope(scope, 'l1_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.abs(weights)),
name=name)
def mn_i(weights, name='maxnorm_i_regularizer'):
"""Applies max-norm regularization to weights."""
with tf.name_scope(name) as scope:
my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale')
if tf.__version__ <= '0.12':
standard_ops_fn = standard_ops.mul
else:
standard_ops_fn = standard_ops.multiply
return standard_ops_fn(
my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope
)
def li(weights, name=None):
"""Applies li regularization to weights."""
# with ops.op_scope([weights], name, 'li_regularizer') as scope: # tf.op_scope(values, name, default_name) is deprecated, use tf.name_scope(name, default_name, values)
try: # TF12
with ops.name_scope(scope, 'li_regularizer', [weights]) as name:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
name=scope)
except: # TF11
with ops.op_scope([weights], name, 'li_regularizer') as scope:
my_scale = ops.convert_to_tensor(scale,
dtype=weights.dtype.base_dtype,
name='scale')
return standard_ops.mul(
my_scale,
standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))),
name=scope)
def _project_log_stochastic_matrix_wrt_kl_divergence(log_matrix):
"""Projects its argument onto the set of log-left-stochastic matrices.
Args:
log_matrix: 2d square tensor, the element-wise logarithm of the matrix to
project.
Returns:
The 2d square tensor that results from projecting exp(`matrix`) onto the set
of left-stochastic matrices w.r.t. the KL-divergence applied column-wise.
"""
# For numerical reasons, make sure that the largest matrix element is zero
# before exponentiating.
log_matrix -= standard_ops.reduce_max(log_matrix, axis=0, keep_dims=True)
log_matrix -= standard_ops.log(
standard_ops.reduce_sum(
standard_ops.exp(log_matrix), axis=0, keep_dims=True))
return log_matrix
def _project_log_stochastic_matrix_wrt_kl_divergence(log_matrix):
"""Projects its argument onto the set of log-left-stochastic matrices.
Args:
log_matrix: 2d square tensor, the element-wise logarithm of the matrix to
project.
Returns:
The 2d square tensor that results from projecting exp(`matrix`) onto the set
of left-stochastic matrices w.r.t. the KL-divergence applied column-wise.
"""
# For numerical reasons, make sure that the largest matrix element is zero
# before exponentiating.
log_matrix -= standard_ops.reduce_max(log_matrix, axis=0, keepdims=True)
log_matrix -= standard_ops.log(
standard_ops.reduce_sum(
standard_ops.exp(log_matrix), axis=0, keepdims=True))
return log_matrix
def l1(weights, name=None):
"""Applies L1 regularization to weights."""
with ops.op_scope([weights], name, "l1_regularizer") as scope:
my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name="scale")
return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=scope)
def _distribution(self, state):
distribution = _maximal_eigenvector_power_method(
self._stochastic_matrix(state))
distribution = standard_ops.abs(distribution)
distribution /= standard_ops.reduce_sum(distribution)
return distribution
def _distribution(self, state):
distribution = _maximal_eigenvector_power_method(
self._stochastic_matrix(state))
distribution = standard_ops.abs(distribution)
distribution /= standard_ops.reduce_sum(distribution)
return distribution
def Body(it, cost): embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0]) cost += tf.reduce_sum(embedding) return it + 1, cost
def Body(it, cost):
embedding = embedding_ops.embedding_lookup(embedding_matrix, [0])
cost = tf.cond(tf.equal(it, 3),
lambda: tf.square(cost),
lambda: cost + tf.reduce_sum(embedding))
return it + 1, cost
