def summarize_activation(op):
"""Summarize an activation.
This applies the given activation and adds useful summaries specific to the
activation.
Args:
op: The tensor to summarize (assumed to be a layer activation).
Returns:
The summary op created to summarize `op`.
"""
if op.op.type in ('Relu', 'Softplus', 'Relu6'):
# Using inputs to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(
standard_ops.less(
op.op.inputs[0],
standard_ops.cast(0.0, op.op.inputs[0].dtype)))),
'%s/zeros' % op.op.name)
if op.op.type == 'Relu6':
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(
standard_ops.greater(
op.op.inputs[0],
standard_ops.cast(6.0, op.op.inputs[0].dtype)))),
'%s/sixes' % op.op.name)
return _add_histogram_summary(op, '%s/activation' % op.op.name)
def _apply_activation_with_summaries(x, activation_fn):
"""Returns activation_fn(x).
This applies the given activation and adds useful summaries specific to the
activation.
Args:
x: The tensor to apply activation to.
activation_fn: An activation function.
Returns:
A tensor with activation applied to x.
"""
if activation_fn is None:
return x
y = activation_fn(x)
if activation_fn in (nn.relu, nn.softplus, nn.relu6):
# Using x for comparison to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(standard_ops.less(x, 0.0))),
'%s/zeros' % y.op.name)
if activation_fn is nn.relu6:
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(standard_ops.greater(x, 6.0))),
'%s/sixes' % y.op.name)
if activation_fn is nn.l2_normalize:
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.sqrt(standard_ops.sum(standard_ops.square(x),
1))),
'%s/length' % y.op.name)
_add_histogram_summary(y, '%s/activations' % y.op.name)
return y
def _apply_activation_with_summaries(x, activation_fn):
"""Returns activation_fn(x).
This applies the given activation and adds useful summaries specific to the
activation.
Args:
x: The tensor to apply activation to.
activation_fn: An activation function.
Returns:
A tensor with activation applied to x.
"""
if activation_fn is None:
return x
y = activation_fn(x)
if activation_fn in (nn.relu, nn.softplus, nn.relu6):
# Using x for comparison to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.to_float(standard_ops.less(
x, 0.0))), '%s/zeros' % y.op.name)
if activation_fn is nn.relu6:
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.to_float(standard_ops.greater(
x, 6.0))), '%s/sixes' % y.op.name)
if activation_fn is nn.l2_normalize:
_add_scalar_summary(
standard_ops.reduce_mean(standard_ops.sqrt(standard_ops.sum(
standard_ops.square(x), 1))), '%s/length' % y.op.name)
_add_histogram_summary(y, '%s/activations' % y.op.name)
return y
def summarize_activation(op):
"""Summarize an activation.
This applies the given activation and adds useful summaries specific to the
activation.
Args:
op: The tensor to summarize (assumed to be a layer activation).
Returns:
The summary op created to summarize `op`.
"""
if op.op.type in ("Relu", "Softplus", "Relu6"):
# Using inputs to avoid floating point equality and/or epsilons.
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(standard_ops.less(op.op.inputs[0], standard_ops.cast(0.0, op.op.inputs[0].dtype)))
),
"%s/zeros" % op.op.name,
)
if op.op.type == "Relu6":
_add_scalar_summary(
standard_ops.reduce_mean(
standard_ops.to_float(
standard_ops.greater(op.op.inputs[0], standard_ops.cast(6.0, op.op.inputs[0].dtype))
)
),
"%s/sixes" % op.op.name,
)
return _add_histogram_summary(op, "%s/activation" % op.op.name)
def while_loop_body(iteration, multipliers, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = standard_ops.minimum(
0.0, (radius - standard_ops.reduce_sum(multipliers)) /
standard_ops.maximum(1.0, standard_ops.reduce_sum(inactive)))
multipliers += scale * inactive
new_inactive = standard_ops.to_float(multipliers > 0)
multipliers *= new_inactive
return (iteration, multipliers, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = (1.0 - standard_ops.reduce_sum(
matrix, axis=0, keepdims=True)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive, axis=0, keepdims=True))
matrix += scale * inactive
new_inactive = standard_ops.to_float(matrix > 0)
matrix *= new_inactive
return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, matrix, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = (1.0 - standard_ops.reduce_sum(
matrix, axis=0, keep_dims=True)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive, axis=0, keep_dims=True))
matrix += scale * inactive
new_inactive = standard_ops.to_float(matrix > 0)
matrix *= new_inactive
return (iteration, matrix, new_inactive, inactive)
def while_loop_body(iteration, multipliers, inactive, old_inactive):
"""Performs one iteration of the projection."""
del old_inactive # Needed by the condition, but not the body.
iteration += 1
scale = standard_ops.minimum(
0.0,
(radius - standard_ops.reduce_sum(multipliers)) / standard_ops.maximum(
1.0, standard_ops.reduce_sum(inactive)))
multipliers += scale * inactive
new_inactive = standard_ops.to_float(multipliers > 0)
multipliers *= new_inactive
return (iteration, multipliers, new_inactive, inactive)
def _projection_op(self, state, name=None):
with ops.colocate_with(state):
# Gets the dimension of the state (num_constraints + 1)--all of these
# assertions are of things that should be impossible, since the state
# passed into this method will have the same shape as that returned by
# _initial_state().
state_shape = state.get_shape()
assert state_shape is not None
assert state_shape.ndims == 2
assert state_shape[0] == state_shape[1]
dimension = state_shape.dims[0].value
assert dimension is not None
minimum_log_multiplier = standard_ops.log(
self._minimum_multiplier_radius / standard_ops.to_float(dimension))
return state_ops.assign(
state,
standard_ops.maximum(
_project_log_stochastic_matrix_wrt_kl_divergence(state),
minimum_log_multiplier),
name=name)
def _projection_op(self, state, name=None):
with ops.colocate_with(state):
# Gets the dimension of the state (num_constraints + 1)--all of these
# assertions are of things that should be impossible, since the state
# passed into this method will have the same shape as that returned by
# _initial_state().
state_shape = state.get_shape()
assert state_shape is not None
assert state_shape.ndims == 2
assert state_shape[0] == state_shape[1]
dimension = state_shape[0].value
assert dimension is not None
minimum_log_multiplier = standard_ops.log(
self._minimum_multiplier_radius / standard_ops.to_float(dimension))
return state_ops.assign(
state,
standard_ops.maximum(
_project_log_stochastic_matrix_wrt_kl_divergence(state),
minimum_log_multiplier),
name=name)
