def compute_step(x_val, geometric=False):
if geometric:
# Consider geometric series where t_mul != 1
# 1 + t_mul + t_mul^2 ... = (1 - t_mul^i_restart) / (1 - t_mul)
# First find how many restarts were performed for a given x_val
# Find maximal integer i_restart value for which this equation holds
# x_val >= (1 - t_mul^i_restart) / (1 - t_mul)
# x_val * (1 - t_mul) <= (1 - t_mul^i_restart)
# t_mul^i_restart <= (1 - x_val * (1 - t_mul))
# tensorflow allows only log with base e
# i_restart <= log(1 - x_val * (1 - t_mul) / log(t_mul)
# Find how many restarts were performed
i_restart = math_ops.floor(
math_ops.log(c_one - x_val * (c_one - t_mul)) / math_ops.log(t_mul))
# Compute the sum of all restarts before the current one
sum_r = (c_one - t_mul ** i_restart) / (c_one - t_mul)
# Compute our position within the current restart
x_val = (x_val - sum_r) / t_mul ** i_restart
else:
# Find how many restarts were performed
i_restart = math_ops.floor(x_val)
# Compute our position within the current restart
x_val = x_val - i_restart
return i_restart, x_val
def compute_step(completed_fraction, geometric=False):
if geometric:
i_restart = math_ops.floor(math_ops.log(1.0 - completed_fraction * (
1.0 - t_mul)) / math_ops.log(t_mul))
sum_r = (1.0 - t_mul ** i_restart) / (1.0 - t_mul)
completed_fraction = (completed_fraction - sum_r) / t_mul ** i_restart
else:
i_restart = math_ops.floor(completed_fraction)
completed_fraction = completed_fraction - i_restart
return i_restart, completed_fraction
def compute_step(completed_fraction, geometric=False):
"""Helper for `cond` operation."""
if geometric:
i_restart = math_ops.floor(
math_ops.log(1.0 - completed_fraction * (1.0 - t_mul)) /
math_ops.log(t_mul))
sum_r = (1.0 - t_mul**i_restart) / (1.0 - t_mul)
completed_fraction = (completed_fraction - sum_r) / t_mul**i_restart
else:
i_restart = math_ops.floor(completed_fraction)
completed_fraction -= i_restart
return i_restart, completed_fraction
def _log_cdf(self, y):
low = self._low
high = self._high
# Recall the promise:
# cdf(y) := P[Y <= y]
# = 1, if y >= high,
# = 0, if y < low,
# = P[X <= y], otherwise.
# P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in
# between.
j = math_ops.floor(y)
result_so_far = self.distribution.log_cdf(j)
# Broadcast, because it's possible that this is a single distribution being
# evaluated on a number of samples, or something like that.
j += array_ops.zeros_like(result_so_far)
# Re-define values at the cutoffs.
if low is not None:
neg_inf = -np.inf * array_ops.ones_like(result_so_far)
result_so_far = array_ops.where(j < low, neg_inf, result_so_far)
if high is not None:
result_so_far = array_ops.where(j >= high,
array_ops.zeros_like(result_so_far),
result_so_far)
return result_so_far
def _cdf(self, y):
lower_cutoff = self._lower_cutoff
upper_cutoff = self._upper_cutoff
# Recall the promise:
# cdf(y) := P[Y <= y]
# = 1, if y >= upper_cutoff,
# = 0, if y < lower_cutoff,
# = P[X <= y], otherwise.
# P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in
# between.
j = math_ops.floor(y)
# P[X <= j], used when lower_cutoff < X < upper_cutoff.
result_so_far = self.distribution.cdf(j)
# Broadcast, because it's possible that this is a single distribution being
# evaluated on a number of samples, or something like that.
j += array_ops.zeros_like(result_so_far)
# Re-define values at the cutoffs.
if lower_cutoff is not None:
result_so_far = math_ops.select(j < lower_cutoff,
array_ops.zeros_like(result_so_far),
result_so_far)
if upper_cutoff is not None:
result_so_far = math_ops.select(j >= upper_cutoff,
array_ops.ones_like(result_so_far),
result_so_far)
return result_so_far
def _log_unnormalized_prob(self, x):
if self.validate_args:
x = distribution_util.embed_check_nonnegative_integer_form(x)
else:
# For consistency with cdf, we take the floor.
x = math_ops.floor(x)
return x * self.log_rate - math_ops.lgamma(1. + x)
def _cdf(self, x):
if self.validate_args:
# We set `check_integer=False` since the CDF is defined on whole real
# line.
x = distribution_util.embed_check_nonnegative_discrete(
x, check_integer=False)
return math_ops.igammac(math_ops.floor(x + 1), self.rate)
def dropout_selu_impl(x, rate, alpha, noise_shape, seed, name):
keep_prob = 1.0 - rate
x = ops.convert_to_tensor(x, name="x")
if isinstance(keep_prob, numbers.Real) and not 0. < keep_prob <= 1.:
raise ValueError("keep_prob must be a scalar tensor or a float in the "
"range (0, 1], got %g" % keep_prob)
keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name="keep_prob")
keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
alpha = ops.convert_to_tensor(alpha, dtype=x.dtype, name="alpha")
keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
if tensor_util.constant_value(keep_prob) == 1:
return x
noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype)
binary_tensor = math_ops.floor(random_tensor)
ret = x * binary_tensor + alpha * (1-binary_tensor)
a = tf.sqrt(fixedPointVar / (keep_prob *((1-keep_prob) * tf.pow(alpha-fixedPointMean,2) + fixedPointVar)))
b = fixedPointMean - a * (keep_prob * fixedPointMean + (1 - keep_prob) * alpha)
ret = a * ret + b
ret.set_shape(x.get_shape())
return ret
def testChi2WithAbsDf(self):
with self.cached_session():
df_v = np.array([-1.3, -3.2, 5], dtype=np.float64)
chi2 = chi2_lib.Chi2WithAbsDf(df=df_v)
self.assertAllClose(
math_ops.floor(math_ops.abs(df_v)).eval(),
chi2.df.eval())
def exponential_decay(learning_rate, global_step, decay_steps, decay_rate,
staircase=False, name=None):
"""Applies exponential decay to the learning rate.
When training a model, it is often recommended to lower the learning rate as
the training progresses. This function applies an exponential decay function
to a provided initial learning rate. It requires a `global_step` value to
compute the decayed learning rate. You can just pass a TensorFlow variable
that you increment at each training step.
The function returns the decayed learning rate. It is computed as:
```python
decayed_learning_rate = learning_rate *
decay_rate ^ (global_step / decay_steps)
```
If the argument `staircase` is `True`, then `global_step /decay_steps` is an
integer division and the decayed learning rate follows a staircase function.
Example: decay every 100000 steps with a base of 0.96:
```python
...
global_step = tf.Variable(0, trainable=False)
starter_learning_rate = 0.1
learning_rate = tf.exponential_decay(starter_learning_rate, global_step,
100000, 0.96, staircase=True)
optimizer = tf.GradientDescent(learning_rate)
# Passing global_step to minimize() will increment it at each step.
optimizer.minimize(...my loss..., global_step=global_step)
```
Args:
learning_rate: A scalar `float32` or `float64` `Tensor` or a
Python number. The initial learning rate.
global_step: A scalar `int32` or `int64` `Tensor` or a Python number.
Global step to use for the decay computation. Must not be negative.
decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number.
Must be positive. See the decay computation above.
decay_rate: A scalar `float32` or `float64` `Tensor` or a
Python number. The decay rate.
staircase: Boolean. It `True` decay the learning rate at discrete intervals.
name: string. Optional name of the operation. Defaults to 'ExponentialDecay'
Returns:
A scalar `Tensor` of the same type as `learning_rate`. The decayed
learning rate.
"""
with ops.op_scope([learning_rate, global_step, decay_steps, decay_rate],
name, "ExponentialDecay") as name:
learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
dtype = learning_rate.dtype
global_step = math_ops.cast(global_step, dtype)
decay_steps = math_ops.cast(decay_steps, dtype)
decay_rate = math_ops.cast(decay_rate, dtype)
p = global_step / decay_steps
if staircase:
p = math_ops.floor(p)
return math_ops.mul(learning_rate, math_ops.pow(decay_rate, p), name=name)
def inverse_time_decay(learning_rate, global_step, decay_steps, decay_rate,
staircase=False, name=None):
"""Applies inverse time decay to the initial learning rate.
When training a model, it is often recommended to lower the learning rate as
the training progresses. This function applies an inverse decay function
to a provided initial learning rate. It requires an `global_step` value to
compute the decayed learning rate. You can just pass a TensorFlow variable
that you increment at each training step.
The function returns the decayed learning rate. It is computed as:
```python
decayed_learning_rate = learning_rate / (1 + decay_rate * t)
```
Example: decay 1/t with a rate of 0.5:
```python
...
global_step = tf.Variable(0, trainable=False)
learning_rate = 0.1
k = 0.5
learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, k)
# Passing global_step to minimize() will increment it at each step.
learning_step = (
tf.train.GradientDescentOptimizer(learning_rate)
.minimize(...my loss..., global_step=global_step)
)
```
Args:
learning_rate: A scalar `float32` or `float64` `Tensor` or a
Python number. The initial learning rate.
global_step: A Python number.
Global step to use for the decay computation. Must not be negative.
decay_rate: A Python number. The decay rate.
name: String. Optional name of the operation. Defaults to
'InverseTimeDecay'
Returns:
A scalar `Tensor` of the same type as `learning_rate`. The decayed
learning rate.
"""
with ops.name_scope(name, "InverseTimeDecay",
[learning_rate, global_step, decay_rate]) as name:
learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
dtype = learning_rate.dtype
global_step = math_ops.cast(global_step, dtype)
decay_steps = math_ops.cast(decay_steps, dtype)
decay_rate = math_ops.cast(decay_rate, dtype)
p = global_step / decay_steps
if staircase:
p = math_ops.floor(p)
const = math_ops.cast(constant_op.constant(1), learning_rate.dtype)
denom = math_ops.add(const, math_ops.mul(decay_rate, p))
return math_ops.div(learning_rate, denom, name=name)
def dropout(self, input_, keep_prob):
with ops.op_scope([input_], None, "dropout") as name:
rands = keep_prob + random_ops.random_uniform(
array_ops.shape(input_))
floored = math_ops.floor(rands)
ret = input_ * math_ops.inv(keep_prob) * floored
ret.set_shape(input_.get_shape())
return ret
def _cdf(self, positive_counts):
if self.validate_args:
positive_counts = math_ops.floor(
distribution_util.embed_check_nonnegative_discrete(
positive_counts, check_integer=False))
return math_ops.betainc(
self.total_count, positive_counts + 1.,
math_ops.sigmoid(-self.logits))
def __init__(self, df, validate_args=False, allow_nan_stats=True,
name="Chi2WithAbsDf"):
with ops.name_scope(name, values=[df]) as ns:
super(Chi2WithAbsDf, self).__init__(
df=math_ops.floor(math_ops.abs(df)),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
def decayed_lr():
"""Helper to recompute learning rate; most helpful in eager-mode."""
global_step_recomp = math_ops.cast(global_step, dtype)
p = global_step_recomp / decay_steps
if staircase:
p = math_ops.floor(p)
return math_ops.multiply(
learning_rate, math_ops.pow(decay_rate, p), name=name)
def _cdf(self, x):
if self.validate_args:
x = distribution_util.embed_check_nonnegative_integer_form(x)
else:
# Whether or not x is integer-form, the following is well-defined.
# However, scipy takes the floor, so we do too.
x = math_ops.floor(x)
return math_ops.igammac(1. + x, self.rate)
def natural_exp_decay(learning_rate, global_step, decay_steps, decay_rate,
staircase=False, name=None):
"""Applies natural exponential decay to the initial learning rate.
When training a model, it is often recommended to lower the learning rate as
the training progresses. This function applies an exponential decay function
to a provided initial learning rate. It requires an `global_step` value to
compute the decayed learning rate. You can just pass a TensorFlow variable
that you increment at each training step.
The function returns the decayed learning rate. It is computed as:
```python
decayed_learning_rate = learning_rate * exp(-decay_rate * global_step)
```
Example: decay exponetially with a base of 0.96:
```python
...
global_step = tf.Variable(0, trainable=False)
learning_rate = 0.1
k = 0.5
learning_rate = tf.train.exponential_time_decay(learning_rate, global_step, k)
# Passing global_step to minimize() will increment it at each step.
learning_step = (
tf.train.GradientDescentOptimizer(learning_rate)
.minimize(...my loss..., global_step=global_step)
)
```
Args:
learning_rate: A scalar `float32` or `float64` `Tensor` or a
Python number. The initial learning rate.
global_step: A Python number.
Global step to use for the decay computation. Must not be negative.
decay_rate: A Python number. The decay rate.
name: String. Optional name of the operation. Defaults to
'ExponentialTimeDecay'
Returns:
A scalar `Tensor` of the same type as `learning_rate`. The decayed
learning rate.
"""
with ops.op_scope([learning_rate, global_step, decay_rate],
name, "NaturalExpDecay") as name:
learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate")
dtype = learning_rate.dtype
global_step = math_ops.cast(global_step, dtype)
decay_steps = math_ops.cast(decay_steps, dtype)
decay_rate = math_ops.cast(decay_rate, dtype)
p = global_step / decay_steps
if staircase:
p = math_ops.floor(p)
exponent = math_ops.exp(math_ops.mul(math_ops.neg(decay_rate), p))
return math_ops.mul(learning_rate, exponent, name=name)
def decayed_lr():
"""Helper to recompute learning rate; most helpful in eager-mode."""
global_step_recomp = math_ops.cast(global_step, dtype)
p = global_step_recomp / decay_steps
if staircase:
p = math_ops.floor(p)
exponent = math_ops.exp(
math_ops.multiply(math_ops.negative(decay_rate), p))
return math_ops.multiply(learning_rate, exponent, name=name)
def decayed_lr():
"""Helper to recompute learning rate; most helpful in eager-mode."""
global_step_recomp = math_ops.cast(global_step, dtype)
p = global_step_recomp / decay_steps
if staircase:
p = math_ops.floor(p)
const = math_ops.cast(constant_op.constant(1), dtype)
denom = math_ops.add(const, math_ops.multiply(decay_rate, p))
return math_ops.div(learning_rate, denom, name=name)
def compute_cdf(values, value_range, **kwargs):
"""Returns the normalized cumulative distribution of the given values tensor.
Uses tf.while_loop to directly compute the cdf of the values. Number of bins
for histogram is fixed at _NBINS=255
Args:
values: Numeric `Tensor`.
value_range: Shape [2] `Tensor` of same `dtype` as `values`
**kwargs: keyword arguments: name
Returns:
A 1-D `Tensor` holding normalized cdf of values.
"""
nbins = _NBINS
name = kwargs.get('name', None)
with ops.name_scope(name, 'cdf', [values, value_range, nbins]):
values = ops.convert_to_tensor(values, name='values')
value_range = ops.convert_to_tensor(value_range, name='value_range')
nbins_float = np.float32(nbins)
# Map tensor values that fall within value_range to [0, 1].
scaled_values = math_ops.truediv(
values - value_range[0],
value_range[1] - value_range[0],
name='scaled_values')
# map tensor values within the open interval value_range to {0,.., nbins-1},
# values outside the open interval will be zero or less, or nbins or more.
indices = math_ops.floor(nbins_float * scaled_values, name='indices')
# Clip edge cases (e.g. value = value_range[1]) or "outliers."
indices = math_ops.cast(
clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32)
cdf = array_ops.zeros(nbins)
i = constant_op.constant(0)
def loop_cond(loop_count, _):
return math_ops.less(loop_count, nbins)
def loop_body(loop_count, cdf):
temp = math_ops.reduce_sum(
math_ops.cast(
math_ops.less_equal(indices, loop_count), dtypes.float32))
cdf = math_ops.add(
cdf,
array_ops.one_hot(
loop_count, depth=_NBINS, on_value=temp, off_value=0.0))
return [loop_count + 1, cdf]
_, cdf = control_flow_ops.while_loop(
loop_cond, loop_body, [i, cdf], maximum_iterations=nbins)
return math_ops.div(cdf, math_ops.reduce_max(cdf))
def testStudentTWithAbsDfSoftplusSigma(self):
with self.test_session():
df = constant_op.constant([-3.2, -4.6])
mu = constant_op.constant([-4.2, 3.4])
sigma = constant_op.constant([-6.4, -8.8])
student = ds.StudentTWithAbsDfSoftplusSigma(df=df, mu=mu, sigma=sigma)
self.assertAllClose(
math_ops.floor(math_ops.abs(df)).eval(), student.df.eval())
self.assertAllClose(mu.eval(), student.mu.eval())
self.assertAllClose(nn_ops.softplus(sigma).eval(), student.sigma.eval())
def _compare(self, x):
np_floor, np_ceil = np.floor(x), np.ceil(x)
with self.cached_session() as sess:
inx = ops.convert_to_tensor(x)
ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx)
tf_floor, tf_ceil = sess.run([ofloor, oceil])
self.assertAllEqual(np_floor, tf_floor)
self.assertAllEqual(np_ceil, tf_ceil)
self.assertShapeEqual(np_floor, ofloor)
self.assertShapeEqual(np_ceil, oceil)
def __init__(self, df, mu, sigma, validate_args=False, allow_nan_stats=True, name="StudentTWithAbsDfSoftplusSigma"):
with ops.name_scope(name, values=[df, mu, sigma]) as ns:
super(StudentTWithAbsDfSoftplusSigma, self).__init__(
df=math_ops.floor(math_ops.abs(df)),
mu=mu,
sigma=nn.softplus(sigma),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns,
)
def _cdf(self, counts):
if self.validate_args:
# We set `check_integer=False` since the CDF is defined on whole real
# line.
counts = math_ops.floor(
distribution_util.embed_check_nonnegative_discrete(
counts, check_integer=False))
counts *= array_ops.ones_like(self.probs)
return array_ops.where(
counts < 0., array_ops.zeros_like(counts), -math_ops.expm1(
(counts + 1) * math_ops.log1p(-self.probs)))
def GenRNNMask(keep_prob=1.0, is_training=False, batch_size=64, inter_dim=1, dtype=None, seed=None):
is_training = ops.convert_to_tensor(is_training, name='is_training')
noise_shape = (batch_size, inter_dim)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape,
seed=seed,
dtype=tf.float32)
binary_tensor = math_ops.floor(random_tensor)
ret = tf.cond(is_training, lambda: tf.identity(binary_tensor),
lambda: tf.identity(keep_prob))
return ret
def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
"""Computes dropout.
With probability `keep_prob`, outputs the input element scaled up by
`1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected
sum is unchanged.
By default, each element is kept or dropped independently. If `noise_shape`
is specified, it must be
[broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]`
will make independent decisions. For example, if `shape(x) = [k, l, m, n]`
and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be
kept independently and each row and column will be kept or not kept together.
Args:
x: A tensor.
keep_prob: A scalar `Tensor` with the same type as x. The probability
that each element is kept.
noise_shape: A 1-D `Tensor` of type `int32`, representing the
shape for randomly generated keep/drop flags.
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
name: A name for this operation (optional).
Returns:
A Tensor of the same shape of `x`.
Raises:
ValueError: If `keep_prob` is not in `(0, 1]`.
"""
with ops.op_scope([x], name, "dropout") as name:
x = ops.convert_to_tensor(x, name="x")
if isinstance(keep_prob, float) and not (0 < keep_prob <= 1):
raise ValueError(
"keep_prob must be a scalar tensor or a float in the "
"range (0, 1], got %g" % keep_prob)
keep_prob = ops.convert_to_tensor(keep_prob,
dtype=x.dtype,
name="keep_prob")
keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
noise_shape = noise_shape or array_ops.shape(x)
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape,
seed=seed,
dtype=x.dtype)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
ret = x * math_ops.inv(keep_prob) * binary_tensor
ret.set_shape(x.get_shape())
return ret
def _input_dropout(self, inputs):
# This implementation of dropout dropouts an entire feature along the time dim
random_tensor = self._keep_prob
random_tensor += random_ops.random_uniform(
[self._batch_size, self._num_inputs], dtype=inputs.dtype)
random_tensor = tf.tile(random_tensor, [1, self._num_unrollings])
binary_tensor = math_ops.floor(random_tensor)
ret = math_ops.div(inputs, self._keep_prob) * binary_tensor
ret.set_shape(inputs.get_shape())
return ret
def _log_prob(self, x):
if self.validate_args:
x = distribution_util.embed_check_nonnegative_integer_form(x)
else:
# For consistency with cdf, we take the floor.
x = math_ops.floor(x)
x *= array_ops.ones_like(self.probs)
probs = self.probs * array_ops.ones_like(x)
safe_domain = array_ops.where(math_ops.equal(x, 0.),
array_ops.zeros_like(probs), probs)
return x * math_ops.log1p(-safe_domain) + math_ops.log(probs)
def _cdf(self, x):
if self.validate_args:
x = distribution_util.embed_check_nonnegative_integer_form(x)
else:
# Whether or not x is integer-form, the following is well-defined.
# However, scipy takes the floor, so we do too.
x = math_ops.floor(x)
x *= array_ops.ones_like(self.probs)
return array_ops.where(
x < 0., array_ops.zeros_like(x), -math_ops.expm1(
(1. + x) * math_ops.log1p(-self.probs)))
def __init__(self,
df,
validate_args=False,
allow_nan_stats=True,
name="Chi2WithAbsDf"):
with ops.name_scope(name, values=[df]) as ns:
super(Chi2WithAbsDf,
self).__init__(df=math_ops.floor(math_ops.abs(df)),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
def _compare(self, x):
np_floor, np_ceil = np.floor(x), np.ceil(x)
inx = ops.convert_to_tensor(x)
ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx)
tf_floor, tf_ceil = self.evaluate([ofloor, oceil])
self.assertAllEqual(np_floor, tf_floor)
self.assertAllEqual(np_ceil, tf_ceil)
self.assertShapeEqual(np_floor, ofloor)
self.assertShapeEqual(np_ceil, oceil)
def _input_dropout(self,inputs):
# This implementation of dropout dropouts an entire feature along the time dim
random_tensor = self._keep_prob
random_tensor += random_ops.random_uniform([self._batch_size,self._num_inputs],
dtype=inputs.dtype)
random_tensor = tf.tile(random_tensor,[1,self._num_unrollings])
binary_tensor = math_ops.floor(random_tensor)
ret = math_ops.div(inputs, self._keep_prob) * binary_tensor
ret.set_shape(inputs.get_shape())
return ret
def _variational_recurrent_dropout_value(
self, index, value, noise, keep_prob):
"""Performs dropout given the pre-calculated noise tensor."""
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob + noise
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
ret = math_ops.div(value, keep_prob) * binary_tensor
ret.set_shape(value.get_shape())
return ret
def _variational_recurrent_dropout_value(self, index, value, noise,
keep_prob):
"""Performs dropout given the pre-calculated noise tensor."""
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob + noise
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
ret = math_ops.div(value, keep_prob) * binary_tensor
ret.set_shape(value.get_shape())
return ret
def testStudentTWithAbsDfSoftplusScale(self):
df = constant_op.constant([-3.2, -4.6])
mu = constant_op.constant([-4.2, 3.4])
sigma = constant_op.constant([-6.4, -8.8])
student = student_t.StudentTWithAbsDfSoftplusScale(
df=df, loc=mu, scale=sigma)
self.assertAllClose(
math_ops.floor(self.evaluate(math_ops.abs(df))),
self.evaluate(student.df))
self.assertAllClose(self.evaluate(mu), self.evaluate(student.loc))
self.assertAllClose(
self.evaluate(nn_ops.softplus(sigma)), self.evaluate(student.scale))
def _cdf(self, x):
if self.validate_args:
x = distribution_util.embed_check_nonnegative_integer_form(x)
else:
# Whether or not x is integer-form, the following is well-defined.
# However, scipy takes the floor, so we do too.
x = math_ops.floor(x)
x *= array_ops.ones_like(self.probs)
return array_ops.where(
x < 0.,
array_ops.zeros_like(x),
-math_ops.expm1((1. + x) * math_ops.log1p(-self.probs)))
def testStudentTWithAbsDfSoftplusScale(self):
df = constant_op.constant([-3.2, -4.6])
mu = constant_op.constant([-4.2, 3.4])
sigma = constant_op.constant([-6.4, -8.8])
student = student_t.StudentTWithAbsDfSoftplusScale(df=df,
loc=mu,
scale=sigma)
self.assertAllClose(math_ops.floor(self.evaluate(math_ops.abs(df))),
self.evaluate(student.df))
self.assertAllClose(self.evaluate(mu), self.evaluate(student.loc))
self.assertAllClose(self.evaluate(nn_ops.softplus(sigma)),
self.evaluate(student.scale))
def dropout(x, keep_prob, noise_shape=None, seed=None, name=None):
"""Computes dropout.
With probability `keep_prob`, outputs the input element scaled up by
`1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected
sum is unchanged.
By default, each element is kept or dropped independently. If `noise_shape`
is specified, it must be
[broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]`
will make independent decisions. For example, if `shape(x) = [k, l, m, n]`
and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be
kept independently and each row and column will be kept or not kept together.
Args:
x: A tensor.
keep_prob: A scalar `Tensor` with the same type as x. The probability
that each element is kept.
noise_shape: A 1-D `Tensor` of type `int32`, representing the
shape for randomly generated keep/drop flags.
seed: A Python integer. Used to create random seeds. See
[`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed)
for behavior.
name: A name for this operation (optional).
Returns:
A Tensor of the same shape of `x`.
Raises:
ValueError: If `keep_prob` is not in `(0, 1]`.
"""
with ops.op_scope([x], name, "dropout") as name:
x = ops.convert_to_tensor(x, name="x")
if isinstance(keep_prob, float) and not 0 < keep_prob <= 1:
raise ValueError("keep_prob must be a scalar tensor or a float in the "
"range (0, 1], got %g" % keep_prob)
keep_prob = ops.convert_to_tensor(keep_prob,
dtype=x.dtype,
name="keep_prob")
keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x)
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape,
seed=seed,
dtype=x.dtype)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
ret = x * math_ops.inv(keep_prob) * binary_tensor
ret.set_shape(x.get_shape())
return ret
def beates_dropout(input_vals, keep_prob = 0.5):
# Dropout
keep_prob = ops.convert_to_tensor(keep_prob, dtype=input_vals.dtype, name="keep_prob")
noise_shape = array_ops.shape(input_vals)
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape, dtype=input_vals.dtype)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
ret = input_vals * binary_tensor
ret.set_shape(input_vals.get_shape())
return ret
def testStudentTWithAbsDfSoftplusScale(self):
with self.test_session():
df = constant_op.constant([-3.2, -4.6])
mu = constant_op.constant([-4.2, 3.4])
sigma = constant_op.constant([-6.4, -8.8])
student = student_t.StudentTWithAbsDfSoftplusScale(df=df,
loc=mu,
scale=sigma)
self.assertAllClose(
math_ops.floor(math_ops.abs(df)).eval(), student.df.eval())
self.assertAllClose(mu.eval(), student.loc.eval())
self.assertAllClose(
nn_ops.softplus(sigma).eval(), student.scale.eval())
def cosine_decay(
learn_rate, # learning rate
epoch, # epoch
batch, # batch epoch
decay, # decay
decay_min_fraction,
alpha, # alpha
epochs,
final_epochs, # finalepoch
delay=0,
name=None,
):
with ops.name_scope(name, "LR_Finetune", [learn_rate, epoch]):
# learning_rate = ops.convert_to_tensor(
# learning_rate, name="initial_learning_rate")
learn_rate = ops.convert_to_tensor(learn_rate,
name="initial_learning_rate")
dtype = tf.float32
learn_rate = math_ops.cast(learn_rate, dtype)
batch = math_ops.cast(batch, dtype)
final_epochs = math_ops.cast(final_epochs, dtype)
alpha = math_ops.cast(alpha, dtype)
decay = math_ops.cast(decay, dtype)
epoch = math_ops.cast(epoch, dtype)
completed_fraction = (epoch - delay) / batch
lam = control_flow_ops.cond(
math_ops.less_equal(epoch, delay),
lambda: learn_rate,
lambda: learn_rate * (decay**math_ops.floor(completed_fraction)),
lambda: learn_rate * tf.math.maximum((decay**math_ops.floor(
completed_fraction)), decay_min_fraction),
)
return control_flow_ops.cond(
math_ops.less_equal(epoch, epochs - final_epochs),
lambda: lam,
lambda: lam * (alpha + (1 - alpha) * (0.5 + 0.5 * math_ops.cos(
(epoch - epochs + final_epochs) / final_epochs * 3.14159))),
)
def __call__(self, step):
with ops.name_scope_v2(self.name or "ExponentialDecay") as name:
initial_learning_rate = ops.convert_to_tensor_v2_with_dispatch(
self.initial_learning_rate, name="initial_learning_rate")
dtype = initial_learning_rate.dtype
decay_steps = math_ops.cast(self.decay_steps, dtype)
decay_rate = math_ops.cast(self.decay_rate, dtype)
global_step_recomp = math_ops.cast(step, dtype)
p = global_step_recomp / decay_steps
if self.staircase:
p = math_ops.floor(p)
return math_ops.multiply(
initial_learning_rate, math_ops.pow(decay_rate, p), name=name)
def cdf(self, x, name="cdf"):
"""Cumulative density function.
Args:
x: Non-negative floating point tensor with dtype `dtype` and whose shape
can be broadcast with `self.lam`.
name: A name for this operation.
Returns:
The CDF of the events.
"""
with ops.name_scope(self.name):
with ops.name_scope(name, values=[self.lam, x]):
x = self._check_x(x, check_integer=False)
return math_ops.igammac(math_ops.floor(x + 1), self.lam)
def __init__(self,
df,
validate_args=False,
allow_nan_stats=True,
name="Chi2WithAbsDf"):
parameters = locals()
with ops.name_scope(name, values=[df]):
super(Chi2WithAbsDf,
self).__init__(df=math_ops.floor(math_ops.abs(df,
name="abs_df"),
name="floor_abs_df"),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name)
self._parameters = parameters
def augm(t, arg):
t = tf.image.random_contrast(t,
lower=1 - arg.contrast_var,
upper=1 + arg.contrast_var)
t = tf.image.random_brightness(t, arg.brightness_var)
t = tf.image.random_saturation(t, 1 - arg.saturation_var,
1 + arg.saturation_var)
t = tf.image.random_hue(t, max_delta=arg.hue_var)
random_tensor = 1. - arg.p_flip + random_ops.random_uniform(shape=[1],
dtype=t.dtype)
binary_tensor = math_ops.floor(random_tensor)
augmented = binary_tensor * t + (1 - binary_tensor) * (1 - t)
return augmented
def __call__(self, step):
with ops.name_scope_v2(self.name or "InverseTimeDecay") as name:
initial_learning_rate = ops.convert_to_tensor_v2_with_dispatch(
self.initial_learning_rate, name="initial_learning_rate")
dtype = initial_learning_rate.dtype
decay_steps = math_ops.cast(self.decay_steps, dtype)
decay_rate = math_ops.cast(self.decay_rate, dtype)
global_step_recomp = math_ops.cast(step, dtype)
p = global_step_recomp / decay_steps
if self.staircase:
p = math_ops.floor(p)
const = math_ops.cast(constant_op.constant(1), dtype)
denom = math_ops.add(const, math_ops.multiply(decay_rate, p))
return math_ops.divide(initial_learning_rate, denom, name=name)
def __init__(self,
df,
mu,
sigma,
validate_args=False,
allow_nan_stats=True,
name="StudentTWithAbsDfSoftplusSigma"):
with ops.name_scope(name, values=[df, mu, sigma]) as ns:
super(StudentTWithAbsDfSoftplusSigma,
self).__init__(df=math_ops.floor(math_ops.abs(df)),
mu=mu,
sigma=nn.softplus(sigma),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
def dropped_inputs_training():
with ops.name_scope("drop_activation_training"):
x = ops.convert_to_tensor(inputs, name="x")
if not x.dtype.is_floating:
raise ValueError(
"x has to be a floating point tensor since it's going to"
" be scaled. Got a %s tensor instead." % x.dtype)
if isinstance(self.p, numbers.Real) and not 0 < self.p <= 1:
raise ValueError(
"p must be a scalar tensor or a float in the "
"range (0, 1], got %g" % self.p)
# Early return is nothin to be dropped
if isinstance(self.p, float) and self.p == 1.:
return nn.relu(x)
if context.executing_eagerly():
if isinstance(self.p, ops.EagerTensor):
if self.p.numpy() == 1:
return nn.relu(x)
else:
p = ops.convert_to_tensor(self.p, dtype=x.dtype, name="p")
p.get_shape().assert_is_compatible_with(
tensor_shape.scalar())
# Do nothing if we know keep_prob == 1
if tensor_util.constant_value(p) == 1:
return nn.relu(x)
noise_shape = array_ops.shape(x)
random_tensor = 1 - p
random_tensor += random_ops.random_uniform(noise_shape,
seed=self.seed,
dtype=x.dtype)
# random_tensor ~ uniform distrib [1 - p, 2 - p), ex: [0.05, 1.05)
binary_tensor = math_ops.floor(random_tensor)
# in binary tensor ~ 5% of are set 1 , 95% are set 0
# drop 95% of the negative part, keep all in the positive part
# old implementation:
# ret = - binary_tensor*nn.relu((-x)) + nn.relu(x)
# new implementation, only 1 relu operation
ret = binary_tensor * x + (1 - binary_tensor) * nn.relu(x)
if not context.executing_eagerly():
ret.set_shape(x.get_shape())
return ret
def call(self, inputs, state):
"""Firing rate model RNN:
new_gate_inputs = dt_over_tau*(W * input + U * state + B)+(1 - dt_over_tau)*gate_inputs.
output = new_state = act(new_gate_inputs)
"""
#apply dropout
keep_prob = random_ops.random_uniform(array_ops.shape(state)) + kp
keep_prob = math_ops.floor(keep_prob)
state = keep_prob * state
#one timestep calculation
state = nn_ops.bias_add(
math_ops.matmul(
array_ops.concat([inputs, self._activation(state)], 1),
self._kernel), self._bias) * 0.1 + 0.9 * state
return state, state
def spatial_drop_noise2(net, p, kernel):
input_shape = net.get_shape().as_list()
noise_shape = np.array([input_shape[0], input_shape[1], input_shape[2], 1],
dtype=np.int64)
random_tensor = 1. - p
random_tensor += tf.random_uniform(noise_shape)
binary_tensor = math_ops.floor(random_tensor)
binary_tensor = slim.max_pool2d(binary_tensor,
kernel,
stride=1,
padding='SAME')
noise_tensor = tf.random_normal(net.get_shape()) * binary_tensor
binary_tensor = tf.ones_like(binary_tensor) - binary_tensor
ret = net * binary_tensor
ret.set_shape(net.get_shape())
return ret + noise_tensor, binary_tensor
def apply_dropout():
keep_prob = 1 - rate
# uniform [keep_prob, 1.0 + keep_prob)
random_tensor = keep_prob
random_tensor += random_ops.random_uniform(noise_shape,
seed=seed,
dtype=x.dtype)
# 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob)
binary_tensor = math_ops.floor(random_tensor)
# save binary tensor to variable
assign_op = binary_tensor_var.assign(binary_tensor)
with tf.control_dependencies([assign_op]):
ret = math_ops.divide(x, keep_prob) * binary_tensor
#ret = tf.Print(ret,["apply dropout and save", _is_reuse_binary_tensor_var])
return ret
def clr(base_lr, max_lr, step_size, clr_iterations, name=None):
if clr_iterations is None:
raise ValueError('global_step is required for clr!')
with ops.name_scope(name, 'clr',
[base_lr, max_lr, step_size, clr_iterations]) as name:
base_lr = ops.convert_to_tensor(base_lr, name="learning_rate")
dtype = base_lr.dtype
max_lr = math_ops.cast(max_lr, dtype)
step_size = math_ops.cast(step_size, dtype)
clr_iterations = math_ops.cast(clr_iterations, dtype)
cycle = math_ops.floor(1 + clr_iterations / (2 * step_size))
x = math_ops.abs(clr_iterations / step_size - 2 * cycle + 1)
return math_ops.add(
base_lr,
math_ops.mul(math_ops.subtract(max_lr, base_lr),
math_ops.maximum(0., math_ops.subtract(1., x))))
def mode(self, name="mode"):
"""Mode of the distribution.
Note that when `lam` is an integer, there are actually two modes.
Namely, `lam` and `lam - 1` are both modes. Here we return
only the larger of the two modes.
Args:
name: Name for the op.
Returns:
mode: `Tensor` of the same type and shape as `lam`.
"""
with ops.name_scope(self.name):
with ops.name_scope(name, values=[self.lam]):
return math_ops.floor(self.lam)
def mode(self, name="mode"):
"""Mode of the distribution.
Note that when `(n + 1) * p` is an integer, there are actually two modes.
Namely, `(n + 1) * p` and `(n + 1) * p - 1` are both modes. Here we return
only the larger of the two modes.
Args:
name: The name for this op.
Returns:
The mode of the Binomial distribution.
"""
with ops.name_scope(self.name):
with ops.op_scope([self._n, self._p], name):
return math_ops.floor((self._n + 1) * self._p)
def testFloorDivide(self):
x = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype(
np.float32).reshape([1, 3, 2])
y = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype(
np.float32).reshape([1, 3, 2])
np_out = np.floor_divide(x, y + 0.1)
with self.session(use_gpu=True) as sess:
inx = ops.convert_to_tensor(x)
iny = ops.convert_to_tensor(y + 0.1)
ofunc = inx / iny
out_func2 = math_ops.floor(ofunc)
tf_out = self.evaluate(out_func2)
self.assertAllClose(np_out, tf_out)
def _sample_n(self, n, seed=None):
# Uniform variates must be sampled from the open-interval `(0, 1)` rather
# than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny`
# because it is the smallest, positive, "normal" number. A "normal" number
# is such that the mantissa has an implicit leading 1. Normal, positive
# numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In
# this case, a subnormal number (i.e., np.nextafter) can cause us to sample
# 0.
sampled = random_ops.random_uniform(
array_ops.concat([[n], array_ops.shape(self._probs)], 0),
minval=np.finfo(self.dtype.as_numpy_dtype).tiny,
maxval=1.,
seed=seed,
dtype=self.dtype)
return math_ops.floor(
math_ops.log(sampled) / math_ops.log1p(-self.probs))
def __init__(self,
df,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="StudentTWithAbsDfSoftplusScale"):
parameters = locals()
with ops.name_scope(name, values=[df, scale]) as ns:
super(StudentTWithAbsDfSoftplusScale, self).__init__(
df=math_ops.floor(math_ops.abs(df)),
loc=loc,
scale=nn.softplus(scale, name="softplus_scale"),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=ns)
self._parameters = parameters
def _histogram(values, value_range, nbins=100, dtype=np.int32, name=None):
"""Return histogram of values.
Given the tensor `values`, this operation returns a rank 1 histogram counting
the number of entries in `values` that fell into every bin. The bins are
equal width and determined by the arguments `value_range` and `nbins`.
Args:
values: Numeric `Tensor`.
value_range: Shape [2] `Tensor` of same `dtype` as `values`.
values <= value_range[0] will be mapped to hist[0],
values >= value_range[1] will be mapped to hist[-1].
nbins: Scalar `int32 Tensor`. Number of histogram bins.
dtype: dtype for returned histogram.
name: A name for this operation (defaults to 'histogram').
Returns:
A 1-D `Tensor` holding histogram of values.
"""
with ops.name_scope(name, 'histogram',
[values, value_range, nbins]) as scope:
values = ops.convert_to_tensor(values, name='values')
values = gen_array_ops.reshape(values, [-1])
value_range = ops.convert_to_tensor(value_range, name='value_range')
nbins = ops.convert_to_tensor(nbins, dtype=np.int32, name='nbins')
nbins_float = math_ops.cast(nbins, values.dtype)
# Map tensor values that fall within value_range to [0, 1].
scaled_values = math_ops.truediv(values - value_range[0],
value_range[1] - value_range[0],
name='scaled_values')
# map tensor values within the open interval value_range to {0,.., nbins-1},
# values outside the open interval will be zero or less, or nbins or more.
indices = math_ops.floor(nbins_float * scaled_values, name='indices')
# Clip edge cases (e.g. value = value_range[1]) or "outliers."
indices = math_ops.cast(
clip_ops.clip_by_value(indices, 0, nbins_float - 1), np.int32)
return math_ops.unsorted_segment_sum(array_ops.ones_like(indices,
dtype=dtype),
indices,
nbins,
name=scope)
