def fucking_deep_gaze_logsumexp(input_tensor,axis=None, keepdims=False,
name=None):
"""
Adaptd from
https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/ops/math_ops.py.
It is the same as the classic logsumexp instead you substact log(N) where N
in the number of tensor over which compute the logsumexp (if you have 10
readout nets, N=10). I don't know why they do this.
"""
keepdims = False if keepdims is None else keepdims
input_tensor = ops.convert_to_tensor(input_tensor)
with ops.name_scope(name, "ReduceLogSumExp", [input_tensor]) as name:
raw_max = tf.reduce_max(input_tensor, axis=axis, keep_dims=True)
my_max = array_ops.stop_gradient( array_ops.where(
gen_math_ops.is_finite(raw_max), raw_max,
array_ops.zeros_like(raw_max)))
result = gen_math_ops.log(
#reduce_sum( # normal logsumexp
tf.reduce_mean( # fuckimg modif from deep_gaze for the output only
gen_math_ops.exp(tf.subtract(input_tensor, my_max)),
axis, keep_dims=keepdims))
if not keepdims:
my_max = array_ops.reshape(my_max, array_ops.shape(result))
result = gen_math_ops.add(result, my_max)
return result
def incr_loss_scale():
new_loss_scale = control_flow_ops.cond(
gen_math_ops.is_finite(self._loss_scale * self._incr_ratio),
lambda: self._loss_scale * self._incr_ratio,
lambda: self._loss_scale)
update_op = state_ops.assign(self._loss_scale, new_loss_scale)
# When loss_scale is updated, both good and bad steps are reset.
return control_flow_ops.group(update_op, self._reset_stats())
def incr_loss_scale():
new_loss_scale = control_flow_ops.cond(
gen_math_ops.is_finite(self._loss_scale * self._incr_ratio),
lambda: self._loss_scale * self._incr_ratio,
lambda: self._loss_scale)
update_op = state_ops.assign(self._loss_scale, new_loss_scale)
# When loss_scale is updated, both good and bad steps are reset.
return control_flow_ops.group(update_op, self._reset_stats())
def get_ix_slices_values_without_nan():
""" Gets the indexed slice values without NaN """
ix_slice_values_without_nans = tf.where(gen_math_ops.is_finite(input_tensor.values),
input_tensor.values,
gen_array_ops.zeros_like(input_tensor.values))
print_op = logging_ops.print_v2('WARNING - Tensor %s has NaN or Inf values. %s' %
(input_tensor.name, name or ''))
with ops.control_dependencies([ix_slice_values_without_nans, print_op]):
return array_ops.identity(ix_slice_values_without_nans)
def ensure_finite(input_tensor, name=None):
""" Replaces NaN and Inf in the input tensor
:param input_tensor: The input tensor to check
:return: The tensor with NaN and Inf replaced with `0`
"""
if not LOG_INF_NAN_TENSORS:
return input_tensor
def get_ix_slices_values_without_nan():
""" Gets the indexed slice values without NaN """
ix_slice_values_without_nans = tf.where(gen_math_ops.is_finite(input_tensor.values),
input_tensor.values,
gen_array_ops.zeros_like(input_tensor.values))
print_op = logging_ops.print_v2('WARNING - Tensor %s has NaN or Inf values. %s' %
(input_tensor.name, name or ''))
with ops.control_dependencies([ix_slice_values_without_nans, print_op]):
return array_ops.identity(ix_slice_values_without_nans)
def get_tensor_without_nan():
""" Gets the tensor without NaN """
tensor_without_nans = tf.where(tf.is_finite(input_tensor), input_tensor, tf.zeros_like(input_tensor))
print_op = logging_ops.print_v2('WARNING - Tensor %s has NaN or Inf values. %s' %
(input_tensor.name, name or ''))
with ops.control_dependencies([tensor_without_nans, print_op]):
return array_ops.identity(tensor_without_nans)
# Tensor
if isinstance(input_tensor, ops.Tensor):
return control_flow_ops.cond(math_ops.reduce_all(gen_math_ops.is_finite(input_tensor)),
true_fn=lambda: input_tensor,
false_fn=get_tensor_without_nan)
# Indexed Slices
if isinstance(input_tensor, ops.IndexedSlices):
values = control_flow_ops.cond(math_ops.reduce_all(gen_math_ops.is_finite(input_tensor.values)),
true_fn=lambda: input_tensor.values,
false_fn=get_ix_slices_values_without_nan)
return ops.IndexedSlices(values=values,
indices=input_tensor.indices,
dense_shape=input_tensor.dense_shape)
# Unknown type
return input_tensor
def softmax_2d(input_tensor, axis=None, keepdims=False, name=None):
"""
Adaptd from
https://gist.github.com/raingo/a5808fe356b8da031837
"""
keepdims = False if keepdims is None else keepdims
input_tensor = ops.convert_to_tensor(input_tensor)
with ops.name_scope(name, "softmax_2d", [input_tensor]) as name:
raw_max = tf.reduce_max(input_tensor, axis=axis, keep_dims=True)
my_max = array_ops.stop_gradient( array_ops.where(
gen_math_ops.is_finite(raw_max), raw_max,
array_ops.zeros_like(raw_max)))
target_exp = gen_math_ops.exp(tf.subtract(input_tensor, my_max))
normalize = tf.reduce_sum(target_exp, axis, keep_dims=True)
softmax = target_exp / normalize
return softmax
def apply_gradients(self, grads_and_vars, global_step=None, name=None):
"""Apply gradients. See base class @{tf.train.Optimizer}."""
grads = [g for (g, _) in grads_and_vars]
is_finite_grad = []
for g in grads:
is_finite_grad.append(math_ops.reduce_all(gen_math_ops.is_finite(g)))
is_overall_finite = math_ops.reduce_all(is_finite_grad)
# Only update gradients when all grads are finite.
def true_apply_gradients_fn():
return self._opt.apply_gradients(grads_and_vars, global_step, name)
update_vars = control_flow_ops.cond(
is_overall_finite, true_apply_gradients_fn, gen_control_flow_ops.no_op)
# Potentially adjust gradient scale in case of finite gradients.
return control_flow_ops.group(
update_vars,
self._loss_scale_manager.update_loss_scale(is_overall_finite))
def _conjugate_gradient(self,
loss,
z,
grads_and_vars,
cg_iter,
fix_first_step=False,
init_deltas=None):
minus_gradient = [g for g, v in grads_and_vars]
variables = [v for g, v in grads_and_vars]
H_vars = [array_ops.zeros_like(g) for g in minus_gradient]
if init_deltas is not None:
H_vars = self._Hv(loss, z, variables, init_deltas, self._damping)
curr_dirs = [g - b for g, b in list(zip(minus_gradient, H_vars))]
curr_residuals = [g - b for g, b in list(zip(minus_gradient, H_vars))]
deltas = [array_ops.zeros_like(g) for g in curr_dirs]
deltas_history = []
residuals_history = []
first_alpha = 1
for i in range(cg_iter):
Hvs = self._Hv(loss, z, variables, curr_dirs, self._damping)
if len(Hvs) != len(variables):
raise ValueError("xs and Hvs must have the same length.")
curr_residuals_flatten = [
gen_array_ops.reshape(v, [-1]) for v in curr_residuals
]
curr_dirs_flatten = [
gen_array_ops.reshape(v, [-1]) for v in curr_dirs
]
Hvs_flatten = [gen_array_ops.reshape(v, [-1]) for v in Hvs]
curr_residuals_concat = array_ops.concat(curr_residuals_flatten, 0)
curr_dirs_concat = array_ops.concat(curr_dirs_flatten, 0)
Hvs_concat = array_ops.concat(Hvs_flatten, 0)
alpha = _dot(curr_residuals_concat, curr_residuals_concat) / _dot(
curr_dirs_concat, Hvs_concat)
alpha = control_flow_ops.cond(
gen_math_ops.is_finite(alpha),
lambda: gen_math_ops.maximum(alpha, 1e-6),
lambda: ops.convert_to_tensor(1.0))
if i == 0 and fix_first_step:
first_alpha = alpha
curr_deltas = [d * (alpha / first_alpha) for d in curr_dirs]
deltas = [d1 + d0 for d0, d1 in list(zip(curr_deltas, deltas))]
deltas_history.append(curr_deltas)
residuals_history.append(curr_residuals)
new_residuals = [
r - alpha * v for r, v in list(zip(curr_residuals, Hvs))
]
new_residuals_flatten = [
gen_array_ops.reshape(v, [-1]) for v in new_residuals
]
new_residuals_concat = array_ops.concat(new_residuals_flatten, 0)
beta = _dot(new_residuals_concat, new_residuals_concat) / _dot(
curr_residuals_concat, curr_residuals_concat)
beta = control_flow_ops.cond(gen_math_ops.is_finite(beta),
lambda: beta,
lambda: ops.convert_to_tensor(0.0))
#beta = gen_math_ops.maximum(beta, 1e-4)
new_dirs = [
r + beta * d for r, d in list(zip(new_residuals, curr_dirs))
]
curr_dirs = new_dirs
curr_residuals = new_residuals
return list(zip(deltas, variables)), deltas_history, residuals_history
def huber_loss(labels,
predictions,
weights=1.0,
delta=1.0,
scope=None,
loss_collection=ops.GraphKeys.LOSSES,
reduction=Reduction.SUM_BY_NONZERO_WEIGHTS):
"""Adds a Huber Loss term to the training procedure.
For each value x in `error=labels-predictions`, the following is calculated:
```
0.5 * x^2 if |x| <= d
0.5 * d^2 + d * (|x| - d) if |x| > d
```
where d is `delta`.
See: https://en.wikipedia.org/wiki/Huber_loss
`weights` acts as a coefficient for the loss. If a scalar is provided, then
the loss is simply scaled by the given value. If `weights` is a tensor of size
`[batch_size]`, then the total loss for each sample of the batch is rescaled
by the corresponding element in the `weights` vector. If the shape of
`weights` matches the shape of `predictions`, then the loss of each
measurable element of `predictions` is scaled by the corresponding value of
`weights`.
Args:
labels: The ground truth output tensor, same dimensions as 'predictions'.
predictions: The predicted outputs.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`labels`, and must be broadcastable to `labels` (i.e., all dimensions must
be either `1`, or the same as the corresponding `losses` dimension).
delta: `float`, the point where the huber loss function
changes from a quadratic to linear.
scope: The scope for the operations performed in computing the loss.
loss_collection: collection to which the loss will be added.
reduction: Type of reduction to apply to loss.
Returns:
Weighted loss float `Tensor`. If `reduction` is `NONE`, this has the same
shape as `labels`; otherwise, it is scalar.
Raises:
ValueError: If the shape of `predictions` doesn't match that of `labels` or
if the shape of `weights` is invalid. Also if `labels` or
`predictions` is None.
@compatibility(eager)
The `loss_collection` argument is ignored when executing eagerly. Consider
holding on to the return value or collecting losses via a `tf.keras.Model`.
@end_compatibility
"""
if labels is None:
raise ValueError("labels must not be None.")
if predictions is None:
raise ValueError("predictions must not be None.")
with ops.name_scope(scope, "huber_loss",
(predictions, labels, weights)) as scope:
if not gen_math_ops.is_finite(delta):
return mean_squared_error(labels, predictions, weights, scope,
loss_collection, reduction)
predictions = math_ops.cast(predictions, dtype=dtypes.float32)
labels = math_ops.cast(labels, dtype=dtypes.float32)
predictions.get_shape().assert_is_compatible_with(labels.get_shape())
error = math_ops.subtract(predictions, labels)
abs_error = math_ops.abs(error)
quadratic = math_ops.minimum(abs_error, delta)
# The following expression is the same in value as
# tf.maximum(abs_error - delta, 0), but importantly the gradient for the
# expression when abs_error == delta is 0 (for tf.maximum it would be 1).
# This is necessary to avoid doubling the gradient, since there is already a
# nonzero contribution to the gradient from the quadratic term.
linear = math_ops.subtract(abs_error, quadratic)
losses = math_ops.add(
math_ops.multiply(
ops.convert_to_tensor(0.5, dtype=quadratic.dtype),
math_ops.multiply(quadratic, quadratic)),
math_ops.multiply(delta, linear))
return compute_weighted_loss(losses,
weights,
scope,
loss_collection,
reduction=reduction)