def apply_gradients(self, grads_and_vars, global_step=None, name=None): gradients = [] # Number of stale gradients. stale_counter = variable_scope.get_variable( "stale_counter", [], initializer=init_ops.zeros_initializer(), trainable=False) def _AcceptGradientOp(): with ops.control_dependencies( [self._opt.apply_gradients( grads_and_vars, global_step=global_step, name=name)]): return gen_array_ops.identity(0.0) def _DropGradientOp(): return gen_array_ops.identity(1.0) for grad_and_var in grads_and_vars: grad = grad_and_var[0] if isinstance(grad, ops.Tensor): gradients.append(grad) else: gradients.append(grad.op) with ops.control_dependencies(gradients), ops.colocate_with(global_step): staleness = gen_array_ops.reshape( global_step - self._local_step, shape=()) conditional_update = stale_counter.assign_add(control_flow_ops.cond( gen_math_ops.less_equal(staleness, self._staleness), _AcceptGradientOp, _DropGradientOp)) summary.scalar( "Gradient staleness percentage", stale_counter / (math_ops.cast(global_step + 1, dtypes.float32))) return conditional_update
def apply_gradients(self, grads_and_vars, global_step=None, name=None): gradients = [] # Number of stale gradients. stale_counter = variable_scope.get_variable( "stale_counter", [], initializer=init_ops.zeros_initializer(), trainable=False) def _AcceptGradientOp(): with ops.control_dependencies([ self._opt.apply_gradients(grads_and_vars, global_step=global_step, name=name) ]): return gen_array_ops.identity(0.0) def _DropGradientOp(): return gen_array_ops.identity(1.0) for grad_and_var in grads_and_vars: grad = grad_and_var[0] if isinstance(grad, ops.Tensor): gradients.append(grad) elif grad is not None: gradients.append(grad.op) with ops.control_dependencies(gradients), ops.colocate_with( global_step): staleness = gen_array_ops.reshape(global_step - self._local_step, shape=()) conditional_update = stale_counter.assign_add( control_flow_ops.cond( gen_math_ops.less_equal(staleness, self._staleness), _AcceptGradientOp, _DropGradientOp)) summary.scalar( "Gradient staleness percentage", stale_counter / (math_ops.cast(global_step + 1, dtypes.float32))) return conditional_update
def gen_crossentropy(y_true, y_pred, q=0.7, k=-1.0): # Filter true values ("y_true") in "y_pred" y_ok = array_ops.boolean_mask(y_pred, gen_math_ops.equal(y_true, 1)) # Conversion for Float64 for valid operations in TensorFlow um = np.float64(1.) q = np.float64(q) if k == -1: # cross entropy loss # mean[ (1-y_ok^q)/q ] return K.mean(math_ops.divide( math_ops.subtract(um, math_ops.pow(y_ok, q)), q), axis=-1) else: # truncated cross entropy loss k = np.float64(k) # if y_ok < k # [ (1-k^q)/q ] (no broadcasting in Where()) # [ (1-y_ok^q)/q ] vfunct = array_ops.where( gen_math_ops.less_equal(y_ok, k), gen_array_ops.fill(array_ops.shape(y_ok), (um - k**q) / q), math_ops.divide(math_ops.subtract(um, math_ops.pow(y_ok, q)), q)) return K.mean(vfunct, axis=-1) # mean [ above values ]