def testIndexedSlicesGradientInCondInWhileLoop(self): with ops.Graph().as_default(): embedding_matrix = tf.get_variable( "embedding_matrix", [5, 5], initializer=tf.random_normal_initializer()) def Cond(it, _): return it < 5 def Body(it, cost): embedding = embedding_ops.embedding_lookup(embedding_matrix, [0]) cost = tf.cond(tf.equal(it, 3), lambda: tf.square(cost), lambda: cost + tf.reduce_sum(embedding)) return it + 1, cost _, cost = control_flow_ops.While( Cond, Body, [tf.constant(0), tf.constant(0.0)]) dynamic_grads = tf.gradients(cost, [embedding_matrix])[0] dynamic_grads = tf.segment_sum(dynamic_grads.values, dynamic_grads.indices) embedding = embedding_ops.embedding_lookup(embedding_matrix, [0]) static = tf.square( tf.reduce_sum(embedding) + tf.reduce_sum(embedding) + tf.reduce_sum(embedding)) + tf.reduce_sum(embedding) static_grads = tf.gradients(static, [embedding_matrix])[0] static_grads = tf.segment_sum(static_grads.values, static_grads.indices) with self.test_session() as sess: sess.run(tf.initialize_all_variables()) self.assertAllEqual(*sess.run([static_grads, dynamic_grads]))
def li(weights, name=None): """Applies li regularization to weights.""" with ops.op_scope([weights], name, 'li_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))), name=scope)
def lo(weights, name=None): """Applies group column regularization to weights.""" with ops.op_scope([weights], name, 'lo_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))), name=scope)
def while_loop_body(iteration, multipliers, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = standard_ops.minimum( 0.0, (radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum(1.0, standard_ops.reduce_sum(inactive))) multipliers = multipliers + (scale * inactive) new_inactive = standard_ops.cast(multipliers > 0, multipliers.dtype) multipliers = multipliers * new_inactive return (iteration, multipliers, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = (1.0 - standard_ops.reduce_sum( matrix, axis=0, keepdims=True)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True)) matrix += scale * inactive new_inactive = standard_ops.cast(matrix > 0, matrix.dtype) matrix *= new_inactive return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = (1.0 - standard_ops.reduce_sum( matrix, axis=0, keepdims=True)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True)) matrix = matrix + (scale * inactive) new_inactive = standard_ops.cast(matrix > 0, matrix.dtype) matrix = matrix * new_inactive return (iteration, matrix, new_inactive, inactive)
def lo(weights, name='lo_regularizer'): """Applies group column regularization to weights.""" with tf.name_scope(name) as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') # if tf.__version__ <= '0.12': # standard_ops_fn = standard_ops.mul # else: standard_ops_fn = standard_ops.multiply return standard_ops_fn( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))), name=scope )
def while_loop_body(iteration, multipliers, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = standard_ops.minimum( 0.0, (radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive))) multipliers += scale * inactive new_inactive = standard_ops.cast(multipliers > 0, multipliers.dtype) multipliers *= new_inactive return (iteration, multipliers, new_inactive, inactive)
def group(weights): """Applies group regularization to weights.""" with tf.name_scope('group_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') standard_ops_fn = standard_ops.multiply return standard_ops_fn(my_scale, standard_ops.reduce_sum( standard_ops.sqrt( standard_ops.reduce_sum( tf.square(weights), 1))), name=scope)
def lo(weights, name='lo_regularizer'): """Applies group column regularization to weights.""" with tf.name_scope(name) as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') if tf.__version__ <= '0.12': standard_ops_fn = standard_ops.mul else: standard_ops_fn = standard_ops.multiply return standard_ops_fn( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))), name=scope)
def li(weights, name=None): """Applies li regularization to weights.""" with tf.name_scope('li_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') # if tf.__version__ <= '0.12': # standard_ops_fn = standard_ops.mul # else: standard_ops_fn = standard_ops.multiply return standard_ops_fn( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))), name=scope)
def mn_i(weights, name=None): """Applies max-norm regularization to weights.""" try: # TF12 with ops.name_scope(scope, 'maxnorm_i_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope) except: # TF11 with ops.op_scope([weights], name, 'maxnorm_i_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def li(weights, name=None): """Applies li regularization to weights.""" with tf.name_scope('li_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') if tf.__version__ <= '0.12': standard_ops_fn = standard_ops.mul else: standard_ops_fn = standard_ops.multiply return standard_ops_fn( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))), name=scope)
def lo(weights, name=None): """Applies group column regularization to weights.""" with ops.op_scope([weights], name, 'lo_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') # return standard_ops.mul( # my_scale, # standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(weights**2, 0))), # name=scope) return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 0))), # standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.reduce_mean(tf.square(weights), 0))), name=scope)
def mn_i(weights, name=None): """Applies max-norm regularization to weights.""" with ops.op_scope([weights], name, 'maxnorm_o_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def testIndexedSlicesWithDynamicShapeGradientInWhileLoop(self): for dtype in [dtypes.float32, dtypes.float64]: with self.test_session() as sess: inputs = tf.placeholder(dtype=dtype) initial_outputs = tf.TensorArray(dtype=dtype, dynamic_size=True, size=1) initial_i = tf.constant(0, dtype=dtypes.int32) def Cond(i, _): return i < tf.size(inputs) # pylint: disable=cell-var-from-loop def Body(i, outputs): x = tf.gather(inputs, i) # pylint: disable=cell-var-from-loop outputs = outputs.write(i, x) return i + 1, outputs _, outputs = tf.while_loop(Cond, Body, [initial_i, initial_outputs]) outputs = tf.reduce_sum(outputs.pack()) r = tf.gradients([outputs], [inputs])[0] grad_wr_inputs = ops.convert_to_tensor(r) o, grad = sess.run([outputs, grad_wr_inputs], feed_dict={inputs: [1, 3, 2]}) self.assertEquals(o, 6) self.assertAllEqual(grad, [1] * 3)
def testIndexedSlicesWithShapeGradientInWhileLoop(self): with self.test_session() as sess: num_steps = 9 inputs = tf.placeholder(dtype="float32", shape=[num_steps]) initial_outputs = tf.TensorArray(dtype="float32", size=num_steps) initial_i = tf.constant(0, dtype="int32") def Cond(i, _): return i < num_steps def Body(i, outputs): x = tf.gather(inputs, i) outputs = outputs.write(i, x) return i + 1, outputs _, outputs = tf.while_loop(Cond, Body, [initial_i, initial_outputs]) outputs = tf.reduce_sum(outputs.pack()) r = tf.gradients([outputs], [inputs])[0] grad_wr_inputs = ops.convert_to_tensor(r) o, grad = sess.run([outputs, grad_wr_inputs], feed_dict={inputs: [4, 6, 0, 7, 0, 0, 1, 2, 0]}) self.assertEquals(o, 20) self.assertAllEqual(grad, [1] * num_steps)
def mn_o(weights, name=None): """Applies max-norm regularization to weights.""" with ops.op_scope([weights], name, 'maxnorm_o_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 0)), name=scope)
def lp(weights): """Applies l2 regularization to weights.""" with ops.name_scope(scope, 'lp_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') reg_loss = standard_ops.reduce_sum(math_ops.pow(math_ops.abs(weights), p)) return standard_ops.multiply(my_scale, reg_loss, name=name)
def li(weights, name=None): """Applies li regularization to weights.""" with ops.op_scope([weights], name, 'li_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') # return standard_ops.mul( # my_scale, # standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(weights**2, 1))), # name=scope) return standard_ops.mul( my_scale, standard_ops.reduce_sum( standard_ops.sqrt( standard_ops.reduce_sum(tf.square(weights), 1))), # standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.reduce_mean(tf.square(weights), 1))), name=scope)
def mn_i(weights, name='maxnorm_i_regularizer'): """Applies max-norm regularization to weights.""" with tf.name_scope(name) as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') if tf.__version__ <= '0.12': standard_ops_fn = standard_ops.mul else: standard_ops_fn = standard_ops.multiply return standard_ops_fn(my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope)
def sum_reg(weights, name=None): """Applies sum regularization to weights.""" with ops.name_scope(scope, 'sum_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.multiply(my_scale, standard_ops.reduce_sum(weights), name=name)
def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.op_scope([weights], scope, 'l1_regularizer') as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul(my_scale, standard_ops.reduce_sum( standard_ops.abs(weights)), name=name)
def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.name_scope(scope, 'l1_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=name)
def mn_i(weights, name='maxnorm_i_regularizer'): """Applies max-norm regularization to weights.""" with tf.name_scope(name) as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') if tf.__version__ <= '0.12': standard_ops_fn = standard_ops.mul else: standard_ops_fn = standard_ops.multiply return standard_ops_fn( my_scale, standard_ops.reduce_sum(standard_ops.reduce_max(standard_ops.abs(weights), 1)), name=scope )
def li(weights, name=None): """Applies li regularization to weights.""" # with ops.op_scope([weights], name, 'li_regularizer') as scope: # tf.op_scope(values, name, default_name) is deprecated, use tf.name_scope(name, default_name, values) try: # TF12 with ops.name_scope(scope, 'li_regularizer', [weights]) as name: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))), name=scope) except: # TF11 with ops.op_scope([weights], name, 'li_regularizer') as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name='scale') return standard_ops.mul( my_scale, standard_ops.reduce_sum(standard_ops.sqrt(standard_ops.reduce_sum(tf.square(weights), 1))), name=scope)
def _project_log_stochastic_matrix_wrt_kl_divergence(log_matrix): """Projects its argument onto the set of log-left-stochastic matrices. Args: log_matrix: 2d square tensor, the element-wise logarithm of the matrix to project. Returns: The 2d square tensor that results from projecting exp(`matrix`) onto the set of left-stochastic matrices w.r.t. the KL-divergence applied column-wise. """ # For numerical reasons, make sure that the largest matrix element is zero # before exponentiating. log_matrix -= standard_ops.reduce_max(log_matrix, axis=0, keep_dims=True) log_matrix -= standard_ops.log( standard_ops.reduce_sum( standard_ops.exp(log_matrix), axis=0, keep_dims=True)) return log_matrix
def _project_log_stochastic_matrix_wrt_kl_divergence(log_matrix): """Projects its argument onto the set of log-left-stochastic matrices. Args: log_matrix: 2d square tensor, the element-wise logarithm of the matrix to project. Returns: The 2d square tensor that results from projecting exp(`matrix`) onto the set of left-stochastic matrices w.r.t. the KL-divergence applied column-wise. """ # For numerical reasons, make sure that the largest matrix element is zero # before exponentiating. log_matrix -= standard_ops.reduce_max(log_matrix, axis=0, keepdims=True) log_matrix -= standard_ops.log( standard_ops.reduce_sum( standard_ops.exp(log_matrix), axis=0, keepdims=True)) return log_matrix
def l1(weights, name=None): """Applies L1 regularization to weights.""" with ops.op_scope([weights], name, "l1_regularizer") as scope: my_scale = ops.convert_to_tensor(scale, dtype=weights.dtype.base_dtype, name="scale") return standard_ops.mul(my_scale, standard_ops.reduce_sum(standard_ops.abs(weights)), name=scope)
def _distribution(self, state): distribution = _maximal_eigenvector_power_method( self._stochastic_matrix(state)) distribution = standard_ops.abs(distribution) distribution /= standard_ops.reduce_sum(distribution) return distribution
def Body(it, cost): embedding = embedding_ops.embedding_lookup(embedding_matrix + 0.0, [0]) cost += tf.reduce_sum(embedding) return it + 1, cost
def Body(it, cost): embedding = embedding_ops.embedding_lookup(embedding_matrix, [0]) cost = tf.cond(tf.equal(it, 3), lambda: tf.square(cost), lambda: cost + tf.reduce_sum(embedding)) return it + 1, cost