def apply_gradients(self, grads_and_vars, global_step=None, name=None):
gradients = []
# Number of stale gradients.
stale_counter = variable_scope.get_variable(
"stale_counter", [],
initializer=init_ops.zeros_initializer(),
trainable=False)
def _AcceptGradientOp():
with ops.control_dependencies(
[self._opt.apply_gradients(
grads_and_vars, global_step=global_step, name=name)]):
return gen_array_ops.identity(0.0)
def _DropGradientOp():
return gen_array_ops.identity(1.0)
for grad_and_var in grads_and_vars:
grad = grad_and_var[0]
if isinstance(grad, ops.Tensor):
gradients.append(grad)
else:
gradients.append(grad.op)
with ops.control_dependencies(gradients), ops.colocate_with(global_step):
staleness = gen_array_ops.reshape(
global_step - self._local_step, shape=())
conditional_update = stale_counter.assign_add(control_flow_ops.cond(
gen_math_ops.less_equal(staleness, self._staleness),
_AcceptGradientOp, _DropGradientOp))
summary.scalar(
"Gradient staleness percentage",
stale_counter / (math_ops.cast(global_step + 1, dtypes.float32)))
return conditional_update
def apply_gradients(self, grads_and_vars, global_step=None, name=None):
gradients = []
# Number of stale gradients.
stale_counter = variable_scope.get_variable(
"stale_counter", [],
initializer=init_ops.zeros_initializer(),
trainable=False)
def _AcceptGradientOp():
with ops.control_dependencies([
self._opt.apply_gradients(grads_and_vars,
global_step=global_step,
name=name)
]):
return gen_array_ops.identity(0.0)
def _DropGradientOp():
return gen_array_ops.identity(1.0)
for grad_and_var in grads_and_vars:
grad = grad_and_var[0]
if isinstance(grad, ops.Tensor):
gradients.append(grad)
elif grad is not None:
gradients.append(grad.op)
with ops.control_dependencies(gradients), ops.colocate_with(
global_step):
staleness = gen_array_ops.reshape(global_step - self._local_step,
shape=())
conditional_update = stale_counter.assign_add(
control_flow_ops.cond(
gen_math_ops.less_equal(staleness, self._staleness),
_AcceptGradientOp, _DropGradientOp))
summary.scalar(
"Gradient staleness percentage",
stale_counter / (math_ops.cast(global_step + 1, dtypes.float32)))
return conditional_update
def gen_crossentropy(y_true, y_pred, q=0.7, k=-1.0):
# Filter true values ("y_true") in "y_pred"
y_ok = array_ops.boolean_mask(y_pred, gen_math_ops.equal(y_true, 1))
# Conversion for Float64 for valid operations in TensorFlow
um = np.float64(1.)
q = np.float64(q)
if k == -1: # cross entropy loss
# mean[ (1-y_ok^q)/q ]
return K.mean(math_ops.divide(
math_ops.subtract(um, math_ops.pow(y_ok, q)), q),
axis=-1)
else: # truncated cross entropy loss
k = np.float64(k)
# if y_ok < k
# [ (1-k^q)/q ] (no broadcasting in Where())
# [ (1-y_ok^q)/q ]
vfunct = array_ops.where(
gen_math_ops.less_equal(y_ok, k),
gen_array_ops.fill(array_ops.shape(y_ok), (um - k**q) / q),
math_ops.divide(math_ops.subtract(um, math_ops.pow(y_ok, q)), q))
return K.mean(vfunct, axis=-1) # mean [ above values ]