def summarize_activation(op): """Summarize an activation. This applies the given activation and adds useful summaries specific to the activation. Args: op: The tensor to summarize (assumed to be a layer activation). Returns: The summary op created to summarize `op`. """ if op.op.type in ('Relu', 'Softplus', 'Relu6'): # Using inputs to avoid floating point equality and/or epsilons. _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float( standard_ops.less( op.op.inputs[0], standard_ops.cast(0.0, op.op.inputs[0].dtype)))), '%s/zeros' % op.op.name) if op.op.type == 'Relu6': _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float( standard_ops.greater( op.op.inputs[0], standard_ops.cast(6.0, op.op.inputs[0].dtype)))), '%s/sixes' % op.op.name) return _add_histogram_summary(op, '%s/activation' % op.op.name)
def _apply_activation_with_summaries(x, activation_fn): """Returns activation_fn(x). This applies the given activation and adds useful summaries specific to the activation. Args: x: The tensor to apply activation to. activation_fn: An activation function. Returns: A tensor with activation applied to x. """ if activation_fn is None: return x y = activation_fn(x) if activation_fn in (nn.relu, nn.softplus, nn.relu6): # Using x for comparison to avoid floating point equality and/or epsilons. _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float(standard_ops.less(x, 0.0))), '%s/zeros' % y.op.name) if activation_fn is nn.relu6: _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float(standard_ops.greater(x, 6.0))), '%s/sixes' % y.op.name) if activation_fn is nn.l2_normalize: _add_scalar_summary( standard_ops.reduce_mean( standard_ops.sqrt(standard_ops.sum(standard_ops.square(x), 1))), '%s/length' % y.op.name) _add_histogram_summary(y, '%s/activations' % y.op.name) return y
def _apply_activation_with_summaries(x, activation_fn): """Returns activation_fn(x). This applies the given activation and adds useful summaries specific to the activation. Args: x: The tensor to apply activation to. activation_fn: An activation function. Returns: A tensor with activation applied to x. """ if activation_fn is None: return x y = activation_fn(x) if activation_fn in (nn.relu, nn.softplus, nn.relu6): # Using x for comparison to avoid floating point equality and/or epsilons. _add_scalar_summary( standard_ops.reduce_mean(standard_ops.to_float(standard_ops.less( x, 0.0))), '%s/zeros' % y.op.name) if activation_fn is nn.relu6: _add_scalar_summary( standard_ops.reduce_mean(standard_ops.to_float(standard_ops.greater( x, 6.0))), '%s/sixes' % y.op.name) if activation_fn is nn.l2_normalize: _add_scalar_summary( standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.sum( standard_ops.square(x), 1))), '%s/length' % y.op.name) _add_histogram_summary(y, '%s/activations' % y.op.name) return y
def summarize_activation(op): """Summarize an activation. This applies the given activation and adds useful summaries specific to the activation. Args: op: The tensor to summarize (assumed to be a layer activation). Returns: The summary op created to summarize `op`. """ if op.op.type in ("Relu", "Softplus", "Relu6"): # Using inputs to avoid floating point equality and/or epsilons. _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float(standard_ops.less(op.op.inputs[0], standard_ops.cast(0.0, op.op.inputs[0].dtype))) ), "%s/zeros" % op.op.name, ) if op.op.type == "Relu6": _add_scalar_summary( standard_ops.reduce_mean( standard_ops.to_float( standard_ops.greater(op.op.inputs[0], standard_ops.cast(6.0, op.op.inputs[0].dtype)) ) ), "%s/sixes" % op.op.name, ) return _add_histogram_summary(op, "%s/activation" % op.op.name)
def while_loop_body(iteration, multipliers, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = standard_ops.minimum( 0.0, (radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum(1.0, standard_ops.reduce_sum(inactive))) multipliers += scale * inactive new_inactive = standard_ops.to_float(multipliers > 0) multipliers *= new_inactive return (iteration, multipliers, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = (1.0 - standard_ops.reduce_sum( matrix, axis=0, keepdims=True)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True)) matrix += scale * inactive new_inactive = standard_ops.to_float(matrix > 0) matrix *= new_inactive return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = (1.0 - standard_ops.reduce_sum( matrix, axis=0, keep_dims=True)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive, axis=0, keep_dims=True)) matrix += scale * inactive new_inactive = standard_ops.to_float(matrix > 0) matrix *= new_inactive return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, multipliers, inactive, old_inactive): """Performs one iteration of the projection.""" del old_inactive # Needed by the condition, but not the body. iteration += 1 scale = standard_ops.minimum( 0.0, (radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum( 1.0, standard_ops.reduce_sum(inactive))) multipliers += scale * inactive new_inactive = standard_ops.to_float(multipliers > 0) multipliers *= new_inactive return (iteration, multipliers, new_inactive, inactive)
def _projection_op(self, state, name=None): with ops.colocate_with(state): # Gets the dimension of the state (num_constraints + 1)--all of these # assertions are of things that should be impossible, since the state # passed into this method will have the same shape as that returned by # _initial_state(). state_shape = state.get_shape() assert state_shape is not None assert state_shape.ndims == 2 assert state_shape[0] == state_shape[1] dimension = state_shape.dims[0].value assert dimension is not None minimum_log_multiplier = standard_ops.log( self._minimum_multiplier_radius / standard_ops.to_float(dimension)) return state_ops.assign( state, standard_ops.maximum( _project_log_stochastic_matrix_wrt_kl_divergence(state), minimum_log_multiplier), name=name)
def _projection_op(self, state, name=None): with ops.colocate_with(state): # Gets the dimension of the state (num_constraints + 1)--all of these # assertions are of things that should be impossible, since the state # passed into this method will have the same shape as that returned by # _initial_state(). state_shape = state.get_shape() assert state_shape is not None assert state_shape.ndims == 2 assert state_shape[0] == state_shape[1] dimension = state_shape[0].value assert dimension is not None minimum_log_multiplier = standard_ops.log( self._minimum_multiplier_radius / standard_ops.to_float(dimension)) return state_ops.assign( state, standard_ops.maximum( _project_log_stochastic_matrix_wrt_kl_divergence(state), minimum_log_multiplier), name=name)