def _get_noise_shape(self, inputs):
# Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`,
# which will override `self.noise_shape`, and allows for custom noise
# shapes with dynamically sized inputs.
if self.noise_shape is None:
return self.noise_shape
return nn_ops._get_noise_shape(inputs, self.noise_shape) # pylint: disable=protected-access
def _get_noise_shape(self, inputs):
# Subclasses of `Dropout` may implement `_get_noise_shape(self, inputs)`,
# which will override `self.noise_shape`, and allows for custom noise
# shapes with dynamically sized inputs.
if self.noise_shape is None:
return self.noise_shape
return nn_ops._get_noise_shape(inputs, self.noise_shape) # pylint: disable=protected-access
def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
"""The gradient for `gelu`.
Args:
x: A tensor with type is float.
keep_prob: A tensor, float, rate of every element reserved.
noise_shape: A 1-D tensor, with type int32, shape of keep/drop what random
generated.
seed: Random seed.
name: Layer name.
Returns:
A tensor.
"""
if context.executing_eagerly():
raise RuntimeError("npu_ops.dropout() is not compatible with "
"eager execution.")
x = ops.convert_to_tensor(x, name="x")
if not x.dtype.is_floating:
raise ValueError("x must be a floating point tensor."
" Got a %s tensor instead." % x.dtype)
if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1.0:
raise ValueError("keep_prob must be a float value or a scalar tensor in the "
"range (0, 1], got %g" % keep_prob)
if isinstance(keep_prob, float) and keep_prob == 1.0:
return x
seed, seed2 = random_seed.get_seed(seed)
noise_shape = _get_noise_shape(x, noise_shape)
gen_out = gen_npu_ops.drop_out_gen_mask(noise_shape, keep_prob, seed, seed2, name)
result = gen_npu_ops.drop_out_do_mask(x, gen_out, keep_prob, name)
return result
def dropped_inputs():
rate = self.rate
noise_shape = self.noise_shape
seed = self.seed
with ops.name_scope(None, "coordinated_dropout", [inputs]) as name:
is_rate_number = isinstance(rate, numbers.Real)
if is_rate_number and (rate < 0 or rate >= 1):
raise ValueError(
"rate must be a scalar tensor or a float in the "
"range [0, 1), got %g" % rate)
x = ops.convert_to_tensor(inputs, name="x")
x_dtype = x.dtype
if not x_dtype.is_floating:
raise ValueError(
"x has to be a floating point tensor since it's going "
"to be scaled. Got a %s tensor instead." % x_dtype)
is_executing_eagerly = context.executing_eagerly()
if not tensor_util.is_tensor(rate):
if is_rate_number:
keep_prob = 1 - rate
scale = 1 / keep_prob
scale = ops.convert_to_tensor(scale, dtype=x_dtype)
ret = gen_math_ops.mul(x, scale)
else:
raise ValueError(
"rate is neither scalar nor scalar tensor %r" %
rate)
else:
rate.get_shape().assert_has_rank(0)
rate_dtype = rate.dtype
if rate_dtype != x_dtype:
if not rate_dtype.is_compatible_with(x_dtype):
raise ValueError(
"Tensor dtype %s is incomptaible with Tensor dtype %s: %r"
% (x_dtype.name, rate_dtype.name, rate))
rate = gen_math_ops.cast(rate, x_dtype, name="rate")
one_tensor = constant_op.constant(1, dtype=x_dtype)
ret = gen_math_ops.real_div(
x, gen_math_ops.sub(one_tensor, rate))
noise_shape = nn_ops._get_noise_shape(x, noise_shape)
# Sample a uniform distribution on [0.0, 1.0) and select values larger
# than rate.
#
# NOTE: Random uniform can only generate 2^23 floats on [1.0, 2.0)
# and subtract 1.0.
random_tensor = random_ops.random_uniform(noise_shape,
seed=seed,
dtype=x_dtype)
# NOTE: if (1.0 + rate) - 1 is equal to rate, then that float is selected,
# hence a >= comparison is used.
keep_mask = random_tensor >= rate
ret = gen_math_ops.mul(ret,
gen_math_ops.cast(keep_mask, x_dtype))
if not is_executing_eagerly:
ret.set_shape(x.get_shape())
return ret, keep_mask
def complex_dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
'''
Implementation of complex dropout based on tf.nn.dropout.
'''
with tf.name_scope(name, "complex_dropout", [x]) as name:
# Early return if nothing needs to be dropped.
if isinstance(keep_prob, float) and keep_prob == 1:
return x
noise_shape = _get_noise_shape(x, noise_shape)
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += tf.random_uniform(
noise_shape, seed=seed, dtype=tf.float32)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = tf.floor(random_tensor)
ret = tf.complex(tf.div(tf.real(x), keep_prob) * binary_tensor,
tf.div(tf.imag(x), keep_prob) * binary_tensor)
return ret
def complex_dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
'''
Implementation of complex dropout based on tf.nn.dropout.
The idea is straightforward, just like its done in the real
case if a complex number is dropped out it is set to zero.
The remaining numbers are scaled according to the keep probability.
'''
with tf.name_scope(name, "complex_dropout", [x]) as name:
# Early return if nothing needs to be dropped.
if isinstance(keep_prob, float) and keep_prob == 1:
return x
noise_shape = _get_noise_shape(x, noise_shape)
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += tf.random_uniform(noise_shape,
seed=seed,
dtype=tf.float32)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = tf.floor(random_tensor)
ret = tf.complex(
tf.div(tf.real(x), keep_prob) * binary_tensor,
tf.div(tf.imag(x), keep_prob) * binary_tensor)
return ret
def _get_noise_shape(self, inputs):
if self.noise_shape is None:
return self.noise_shape
return nn_ops._get_noise_shape(inputs, self.noise_shape)
