def compute_step(x_val, geometric=False): if geometric: # Consider geometric series where t_mul != 1 # 1 + t_mul + t_mul^2 ... = (1 - t_mul^i_restart) / (1 - t_mul) # First find how many restarts were performed for a given x_val # Find maximal integer i_restart value for which this equation holds # x_val >= (1 - t_mul^i_restart) / (1 - t_mul) # x_val * (1 - t_mul) <= (1 - t_mul^i_restart) # t_mul^i_restart <= (1 - x_val * (1 - t_mul)) # tensorflow allows only log with base e # i_restart <= log(1 - x_val * (1 - t_mul) / log(t_mul) # Find how many restarts were performed i_restart = math_ops.floor( math_ops.log(c_one - x_val * (c_one - t_mul)) / math_ops.log(t_mul)) # Compute the sum of all restarts before the current one sum_r = (c_one - t_mul ** i_restart) / (c_one - t_mul) # Compute our position within the current restart x_val = (x_val - sum_r) / t_mul ** i_restart else: # Find how many restarts were performed i_restart = math_ops.floor(x_val) # Compute our position within the current restart x_val = x_val - i_restart return i_restart, x_val
def compute_step(completed_fraction, geometric=False): if geometric: i_restart = math_ops.floor(math_ops.log(1.0 - completed_fraction * ( 1.0 - t_mul)) / math_ops.log(t_mul)) sum_r = (1.0 - t_mul ** i_restart) / (1.0 - t_mul) completed_fraction = (completed_fraction - sum_r) / t_mul ** i_restart else: i_restart = math_ops.floor(completed_fraction) completed_fraction = completed_fraction - i_restart return i_restart, completed_fraction
def compute_step(completed_fraction, geometric=False): """Helper for `cond` operation.""" if geometric: i_restart = math_ops.floor( math_ops.log(1.0 - completed_fraction * (1.0 - t_mul)) / math_ops.log(t_mul)) sum_r = (1.0 - t_mul**i_restart) / (1.0 - t_mul) completed_fraction = (completed_fraction - sum_r) / t_mul**i_restart else: i_restart = math_ops.floor(completed_fraction) completed_fraction -= i_restart return i_restart, completed_fraction
def _log_cdf(self, y): low = self._low high = self._high # Recall the promise: # cdf(y) := P[Y <= y] # = 1, if y >= high, # = 0, if y < low, # = P[X <= y], otherwise. # P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in # between. j = math_ops.floor(y) result_so_far = self.distribution.log_cdf(j) # Broadcast, because it's possible that this is a single distribution being # evaluated on a number of samples, or something like that. j += array_ops.zeros_like(result_so_far) # Re-define values at the cutoffs. if low is not None: neg_inf = -np.inf * array_ops.ones_like(result_so_far) result_so_far = array_ops.where(j < low, neg_inf, result_so_far) if high is not None: result_so_far = array_ops.where(j >= high, array_ops.zeros_like(result_so_far), result_so_far) return result_so_far
def _cdf(self, y): lower_cutoff = self._lower_cutoff upper_cutoff = self._upper_cutoff # Recall the promise: # cdf(y) := P[Y <= y] # = 1, if y >= upper_cutoff, # = 0, if y < lower_cutoff, # = P[X <= y], otherwise. # P[Y <= j] = P[floor(Y) <= j] since mass is only at integers, not in # between. j = math_ops.floor(y) # P[X <= j], used when lower_cutoff < X < upper_cutoff. result_so_far = self.distribution.cdf(j) # Broadcast, because it's possible that this is a single distribution being # evaluated on a number of samples, or something like that. j += array_ops.zeros_like(result_so_far) # Re-define values at the cutoffs. if lower_cutoff is not None: result_so_far = math_ops.select(j < lower_cutoff, array_ops.zeros_like(result_so_far), result_so_far) if upper_cutoff is not None: result_so_far = math_ops.select(j >= upper_cutoff, array_ops.ones_like(result_so_far), result_so_far) return result_so_far
def _log_unnormalized_prob(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) else: # For consistency with cdf, we take the floor. x = math_ops.floor(x) return x * self.log_rate - math_ops.lgamma(1. + x)
def _cdf(self, x): if self.validate_args: # We set `check_integer=False` since the CDF is defined on whole real # line. x = distribution_util.embed_check_nonnegative_discrete( x, check_integer=False) return math_ops.igammac(math_ops.floor(x + 1), self.rate)
def dropout_selu_impl(x, rate, alpha, noise_shape, seed, name): keep_prob = 1.0 - rate x = ops.convert_to_tensor(x, name="x") if isinstance(keep_prob, numbers.Real) and not 0. < keep_prob <= 1.: raise ValueError("keep_prob must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep_prob) keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name="keep_prob") keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar()) alpha = ops.convert_to_tensor(alpha, dtype=x.dtype, name="alpha") keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar()) if tensor_util.constant_value(keep_prob) == 1: return x noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype) binary_tensor = math_ops.floor(random_tensor) ret = x * binary_tensor + alpha * (1-binary_tensor) a = tf.sqrt(fixedPointVar / (keep_prob *((1-keep_prob) * tf.pow(alpha-fixedPointMean,2) + fixedPointVar))) b = fixedPointMean - a * (keep_prob * fixedPointMean + (1 - keep_prob) * alpha) ret = a * ret + b ret.set_shape(x.get_shape()) return ret
def testChi2WithAbsDf(self): with self.cached_session(): df_v = np.array([-1.3, -3.2, 5], dtype=np.float64) chi2 = chi2_lib.Chi2WithAbsDf(df=df_v) self.assertAllClose( math_ops.floor(math_ops.abs(df_v)).eval(), chi2.df.eval())
def exponential_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False, name=None): """Applies exponential decay to the learning rate. When training a model, it is often recommended to lower the learning rate as the training progresses. This function applies an exponential decay function to a provided initial learning rate. It requires a `global_step` value to compute the decayed learning rate. You can just pass a TensorFlow variable that you increment at each training step. The function returns the decayed learning rate. It is computed as: ```python decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps) ``` If the argument `staircase` is `True`, then `global_step /decay_steps` is an integer division and the decayed learning rate follows a staircase function. Example: decay every 100000 steps with a base of 0.96: ```python ... global_step = tf.Variable(0, trainable=False) starter_learning_rate = 0.1 learning_rate = tf.exponential_decay(starter_learning_rate, global_step, 100000, 0.96, staircase=True) optimizer = tf.GradientDescent(learning_rate) # Passing global_step to minimize() will increment it at each step. optimizer.minimize(...my loss..., global_step=global_step) ``` Args: learning_rate: A scalar `float32` or `float64` `Tensor` or a Python number. The initial learning rate. global_step: A scalar `int32` or `int64` `Tensor` or a Python number. Global step to use for the decay computation. Must not be negative. decay_steps: A scalar `int32` or `int64` `Tensor` or a Python number. Must be positive. See the decay computation above. decay_rate: A scalar `float32` or `float64` `Tensor` or a Python number. The decay rate. staircase: Boolean. It `True` decay the learning rate at discrete intervals. name: string. Optional name of the operation. Defaults to 'ExponentialDecay' Returns: A scalar `Tensor` of the same type as `learning_rate`. The decayed learning rate. """ with ops.op_scope([learning_rate, global_step, decay_steps, decay_rate], name, "ExponentialDecay") as name: learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") dtype = learning_rate.dtype global_step = math_ops.cast(global_step, dtype) decay_steps = math_ops.cast(decay_steps, dtype) decay_rate = math_ops.cast(decay_rate, dtype) p = global_step / decay_steps if staircase: p = math_ops.floor(p) return math_ops.mul(learning_rate, math_ops.pow(decay_rate, p), name=name)
def inverse_time_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False, name=None): """Applies inverse time decay to the initial learning rate. When training a model, it is often recommended to lower the learning rate as the training progresses. This function applies an inverse decay function to a provided initial learning rate. It requires an `global_step` value to compute the decayed learning rate. You can just pass a TensorFlow variable that you increment at each training step. The function returns the decayed learning rate. It is computed as: ```python decayed_learning_rate = learning_rate / (1 + decay_rate * t) ``` Example: decay 1/t with a rate of 0.5: ```python ... global_step = tf.Variable(0, trainable=False) learning_rate = 0.1 k = 0.5 learning_rate = tf.train.inverse_time_decay(learning_rate, global_step, k) # Passing global_step to minimize() will increment it at each step. learning_step = ( tf.train.GradientDescentOptimizer(learning_rate) .minimize(...my loss..., global_step=global_step) ) ``` Args: learning_rate: A scalar `float32` or `float64` `Tensor` or a Python number. The initial learning rate. global_step: A Python number. Global step to use for the decay computation. Must not be negative. decay_rate: A Python number. The decay rate. name: String. Optional name of the operation. Defaults to 'InverseTimeDecay' Returns: A scalar `Tensor` of the same type as `learning_rate`. The decayed learning rate. """ with ops.name_scope(name, "InverseTimeDecay", [learning_rate, global_step, decay_rate]) as name: learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") dtype = learning_rate.dtype global_step = math_ops.cast(global_step, dtype) decay_steps = math_ops.cast(decay_steps, dtype) decay_rate = math_ops.cast(decay_rate, dtype) p = global_step / decay_steps if staircase: p = math_ops.floor(p) const = math_ops.cast(constant_op.constant(1), learning_rate.dtype) denom = math_ops.add(const, math_ops.mul(decay_rate, p)) return math_ops.div(learning_rate, denom, name=name)
def dropout(self, input_, keep_prob): with ops.op_scope([input_], None, "dropout") as name: rands = keep_prob + random_ops.random_uniform( array_ops.shape(input_)) floored = math_ops.floor(rands) ret = input_ * math_ops.inv(keep_prob) * floored ret.set_shape(input_.get_shape()) return ret
def _cdf(self, positive_counts): if self.validate_args: positive_counts = math_ops.floor( distribution_util.embed_check_nonnegative_discrete( positive_counts, check_integer=False)) return math_ops.betainc( self.total_count, positive_counts + 1., math_ops.sigmoid(-self.logits))
def __init__(self, df, validate_args=False, allow_nan_stats=True, name="Chi2WithAbsDf"): with ops.name_scope(name, values=[df]) as ns: super(Chi2WithAbsDf, self).__init__( df=math_ops.floor(math_ops.abs(df)), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns)
def decayed_lr(): """Helper to recompute learning rate; most helpful in eager-mode.""" global_step_recomp = math_ops.cast(global_step, dtype) p = global_step_recomp / decay_steps if staircase: p = math_ops.floor(p) return math_ops.multiply( learning_rate, math_ops.pow(decay_rate, p), name=name)
def _cdf(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) else: # Whether or not x is integer-form, the following is well-defined. # However, scipy takes the floor, so we do too. x = math_ops.floor(x) return math_ops.igammac(1. + x, self.rate)
def natural_exp_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False, name=None): """Applies natural exponential decay to the initial learning rate. When training a model, it is often recommended to lower the learning rate as the training progresses. This function applies an exponential decay function to a provided initial learning rate. It requires an `global_step` value to compute the decayed learning rate. You can just pass a TensorFlow variable that you increment at each training step. The function returns the decayed learning rate. It is computed as: ```python decayed_learning_rate = learning_rate * exp(-decay_rate * global_step) ``` Example: decay exponetially with a base of 0.96: ```python ... global_step = tf.Variable(0, trainable=False) learning_rate = 0.1 k = 0.5 learning_rate = tf.train.exponential_time_decay(learning_rate, global_step, k) # Passing global_step to minimize() will increment it at each step. learning_step = ( tf.train.GradientDescentOptimizer(learning_rate) .minimize(...my loss..., global_step=global_step) ) ``` Args: learning_rate: A scalar `float32` or `float64` `Tensor` or a Python number. The initial learning rate. global_step: A Python number. Global step to use for the decay computation. Must not be negative. decay_rate: A Python number. The decay rate. name: String. Optional name of the operation. Defaults to 'ExponentialTimeDecay' Returns: A scalar `Tensor` of the same type as `learning_rate`. The decayed learning rate. """ with ops.op_scope([learning_rate, global_step, decay_rate], name, "NaturalExpDecay") as name: learning_rate = ops.convert_to_tensor(learning_rate, name="learning_rate") dtype = learning_rate.dtype global_step = math_ops.cast(global_step, dtype) decay_steps = math_ops.cast(decay_steps, dtype) decay_rate = math_ops.cast(decay_rate, dtype) p = global_step / decay_steps if staircase: p = math_ops.floor(p) exponent = math_ops.exp(math_ops.mul(math_ops.neg(decay_rate), p)) return math_ops.mul(learning_rate, exponent, name=name)
def decayed_lr(): """Helper to recompute learning rate; most helpful in eager-mode.""" global_step_recomp = math_ops.cast(global_step, dtype) p = global_step_recomp / decay_steps if staircase: p = math_ops.floor(p) exponent = math_ops.exp( math_ops.multiply(math_ops.negative(decay_rate), p)) return math_ops.multiply(learning_rate, exponent, name=name)
def decayed_lr(): """Helper to recompute learning rate; most helpful in eager-mode.""" global_step_recomp = math_ops.cast(global_step, dtype) p = global_step_recomp / decay_steps if staircase: p = math_ops.floor(p) const = math_ops.cast(constant_op.constant(1), dtype) denom = math_ops.add(const, math_ops.multiply(decay_rate, p)) return math_ops.div(learning_rate, denom, name=name)
def compute_cdf(values, value_range, **kwargs): """Returns the normalized cumulative distribution of the given values tensor. Uses tf.while_loop to directly compute the cdf of the values. Number of bins for histogram is fixed at _NBINS=255 Args: values: Numeric `Tensor`. value_range: Shape [2] `Tensor` of same `dtype` as `values` **kwargs: keyword arguments: name Returns: A 1-D `Tensor` holding normalized cdf of values. """ nbins = _NBINS name = kwargs.get('name', None) with ops.name_scope(name, 'cdf', [values, value_range, nbins]): values = ops.convert_to_tensor(values, name='values') value_range = ops.convert_to_tensor(value_range, name='value_range') nbins_float = np.float32(nbins) # Map tensor values that fall within value_range to [0, 1]. scaled_values = math_ops.truediv( values - value_range[0], value_range[1] - value_range[0], name='scaled_values') # map tensor values within the open interval value_range to {0,.., nbins-1}, # values outside the open interval will be zero or less, or nbins or more. indices = math_ops.floor(nbins_float * scaled_values, name='indices') # Clip edge cases (e.g. value = value_range[1]) or "outliers." indices = math_ops.cast( clip_ops.clip_by_value(indices, 0, nbins_float - 1), dtypes.int32) cdf = array_ops.zeros(nbins) i = constant_op.constant(0) def loop_cond(loop_count, _): return math_ops.less(loop_count, nbins) def loop_body(loop_count, cdf): temp = math_ops.reduce_sum( math_ops.cast( math_ops.less_equal(indices, loop_count), dtypes.float32)) cdf = math_ops.add( cdf, array_ops.one_hot( loop_count, depth=_NBINS, on_value=temp, off_value=0.0)) return [loop_count + 1, cdf] _, cdf = control_flow_ops.while_loop( loop_cond, loop_body, [i, cdf], maximum_iterations=nbins) return math_ops.div(cdf, math_ops.reduce_max(cdf))
def testStudentTWithAbsDfSoftplusSigma(self): with self.test_session(): df = constant_op.constant([-3.2, -4.6]) mu = constant_op.constant([-4.2, 3.4]) sigma = constant_op.constant([-6.4, -8.8]) student = ds.StudentTWithAbsDfSoftplusSigma(df=df, mu=mu, sigma=sigma) self.assertAllClose( math_ops.floor(math_ops.abs(df)).eval(), student.df.eval()) self.assertAllClose(mu.eval(), student.mu.eval()) self.assertAllClose(nn_ops.softplus(sigma).eval(), student.sigma.eval())
def _compare(self, x): np_floor, np_ceil = np.floor(x), np.ceil(x) with self.cached_session() as sess: inx = ops.convert_to_tensor(x) ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx) tf_floor, tf_ceil = sess.run([ofloor, oceil]) self.assertAllEqual(np_floor, tf_floor) self.assertAllEqual(np_ceil, tf_ceil) self.assertShapeEqual(np_floor, ofloor) self.assertShapeEqual(np_ceil, oceil)
def __init__(self, df, mu, sigma, validate_args=False, allow_nan_stats=True, name="StudentTWithAbsDfSoftplusSigma"): with ops.name_scope(name, values=[df, mu, sigma]) as ns: super(StudentTWithAbsDfSoftplusSigma, self).__init__( df=math_ops.floor(math_ops.abs(df)), mu=mu, sigma=nn.softplus(sigma), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns, )
def _cdf(self, counts): if self.validate_args: # We set `check_integer=False` since the CDF is defined on whole real # line. counts = math_ops.floor( distribution_util.embed_check_nonnegative_discrete( counts, check_integer=False)) counts *= array_ops.ones_like(self.probs) return array_ops.where( counts < 0., array_ops.zeros_like(counts), -math_ops.expm1( (counts + 1) * math_ops.log1p(-self.probs)))
def GenRNNMask(keep_prob=1.0, is_training=False, batch_size=64, inter_dim=1, dtype=None, seed=None): is_training = ops.convert_to_tensor(is_training, name='is_training') noise_shape = (batch_size, inter_dim) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=tf.float32) binary_tensor = math_ops.floor(random_tensor) ret = tf.cond(is_training, lambda: tf.identity(binary_tensor), lambda: tf.identity(keep_prob)) return ret
def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): """Computes dropout. With probability `keep_prob`, outputs the input element scaled up by `1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected sum is unchanged. By default, each element is kept or dropped independently. If `noise_shape` is specified, it must be [broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]` will make independent decisions. For example, if `shape(x) = [k, l, m, n]` and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be kept independently and each row and column will be kept or not kept together. Args: x: A tensor. keep_prob: A scalar `Tensor` with the same type as x. The probability that each element is kept. noise_shape: A 1-D `Tensor` of type `int32`, representing the shape for randomly generated keep/drop flags. seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. name: A name for this operation (optional). Returns: A Tensor of the same shape of `x`. Raises: ValueError: If `keep_prob` is not in `(0, 1]`. """ with ops.op_scope([x], name, "dropout") as name: x = ops.convert_to_tensor(x, name="x") if isinstance(keep_prob, float) and not (0 < keep_prob <= 1): raise ValueError( "keep_prob must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep_prob) keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name="keep_prob") keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar()) noise_shape = noise_shape or array_ops.shape(x) # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype) # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) ret = x * math_ops.inv(keep_prob) * binary_tensor ret.set_shape(x.get_shape()) return ret
def _input_dropout(self, inputs): # This implementation of dropout dropouts an entire feature along the time dim random_tensor = self._keep_prob random_tensor += random_ops.random_uniform( [self._batch_size, self._num_inputs], dtype=inputs.dtype) random_tensor = tf.tile(random_tensor, [1, self._num_unrollings]) binary_tensor = math_ops.floor(random_tensor) ret = math_ops.div(inputs, self._keep_prob) * binary_tensor ret.set_shape(inputs.get_shape()) return ret
def _log_prob(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) else: # For consistency with cdf, we take the floor. x = math_ops.floor(x) x *= array_ops.ones_like(self.probs) probs = self.probs * array_ops.ones_like(x) safe_domain = array_ops.where(math_ops.equal(x, 0.), array_ops.zeros_like(probs), probs) return x * math_ops.log1p(-safe_domain) + math_ops.log(probs)
def _cdf(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) else: # Whether or not x is integer-form, the following is well-defined. # However, scipy takes the floor, so we do too. x = math_ops.floor(x) x *= array_ops.ones_like(self.probs) return array_ops.where( x < 0., array_ops.zeros_like(x), -math_ops.expm1( (1. + x) * math_ops.log1p(-self.probs)))
def __init__(self, df, validate_args=False, allow_nan_stats=True, name="Chi2WithAbsDf"): with ops.name_scope(name, values=[df]) as ns: super(Chi2WithAbsDf, self).__init__(df=math_ops.floor(math_ops.abs(df)), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns)
def _compare(self, x): np_floor, np_ceil = np.floor(x), np.ceil(x) inx = ops.convert_to_tensor(x) ofloor, oceil = math_ops.floor(inx), math_ops.ceil(inx) tf_floor, tf_ceil = self.evaluate([ofloor, oceil]) self.assertAllEqual(np_floor, tf_floor) self.assertAllEqual(np_ceil, tf_ceil) self.assertShapeEqual(np_floor, ofloor) self.assertShapeEqual(np_ceil, oceil)
def _input_dropout(self,inputs): # This implementation of dropout dropouts an entire feature along the time dim random_tensor = self._keep_prob random_tensor += random_ops.random_uniform([self._batch_size,self._num_inputs], dtype=inputs.dtype) random_tensor = tf.tile(random_tensor,[1,self._num_unrollings]) binary_tensor = math_ops.floor(random_tensor) ret = math_ops.div(inputs, self._keep_prob) * binary_tensor ret.set_shape(inputs.get_shape()) return ret
def _variational_recurrent_dropout_value( self, index, value, noise, keep_prob): """Performs dropout given the pre-calculated noise tensor.""" # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob + noise # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) ret = math_ops.div(value, keep_prob) * binary_tensor ret.set_shape(value.get_shape()) return ret
def _variational_recurrent_dropout_value(self, index, value, noise, keep_prob): """Performs dropout given the pre-calculated noise tensor.""" # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob + noise # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) ret = math_ops.div(value, keep_prob) * binary_tensor ret.set_shape(value.get_shape()) return ret
def testStudentTWithAbsDfSoftplusScale(self): df = constant_op.constant([-3.2, -4.6]) mu = constant_op.constant([-4.2, 3.4]) sigma = constant_op.constant([-6.4, -8.8]) student = student_t.StudentTWithAbsDfSoftplusScale( df=df, loc=mu, scale=sigma) self.assertAllClose( math_ops.floor(self.evaluate(math_ops.abs(df))), self.evaluate(student.df)) self.assertAllClose(self.evaluate(mu), self.evaluate(student.loc)) self.assertAllClose( self.evaluate(nn_ops.softplus(sigma)), self.evaluate(student.scale))
def _cdf(self, x): if self.validate_args: x = distribution_util.embed_check_nonnegative_integer_form(x) else: # Whether or not x is integer-form, the following is well-defined. # However, scipy takes the floor, so we do too. x = math_ops.floor(x) x *= array_ops.ones_like(self.probs) return array_ops.where( x < 0., array_ops.zeros_like(x), -math_ops.expm1((1. + x) * math_ops.log1p(-self.probs)))
def testStudentTWithAbsDfSoftplusScale(self): df = constant_op.constant([-3.2, -4.6]) mu = constant_op.constant([-4.2, 3.4]) sigma = constant_op.constant([-6.4, -8.8]) student = student_t.StudentTWithAbsDfSoftplusScale(df=df, loc=mu, scale=sigma) self.assertAllClose(math_ops.floor(self.evaluate(math_ops.abs(df))), self.evaluate(student.df)) self.assertAllClose(self.evaluate(mu), self.evaluate(student.loc)) self.assertAllClose(self.evaluate(nn_ops.softplus(sigma)), self.evaluate(student.scale))
def dropout(x, keep_prob, noise_shape=None, seed=None, name=None): """Computes dropout. With probability `keep_prob`, outputs the input element scaled up by `1 / keep_prob`, otherwise outputs `0`. The scaling is so that the expected sum is unchanged. By default, each element is kept or dropped independently. If `noise_shape` is specified, it must be [broadcastable](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) to the shape of `x`, and only dimensions with `noise_shape[i] == shape(x)[i]` will make independent decisions. For example, if `shape(x) = [k, l, m, n]` and `noise_shape = [k, 1, 1, n]`, each batch and channel component will be kept independently and each row and column will be kept or not kept together. Args: x: A tensor. keep_prob: A scalar `Tensor` with the same type as x. The probability that each element is kept. noise_shape: A 1-D `Tensor` of type `int32`, representing the shape for randomly generated keep/drop flags. seed: A Python integer. Used to create random seeds. See [`set_random_seed`](../../api_docs/python/constant_op.md#set_random_seed) for behavior. name: A name for this operation (optional). Returns: A Tensor of the same shape of `x`. Raises: ValueError: If `keep_prob` is not in `(0, 1]`. """ with ops.op_scope([x], name, "dropout") as name: x = ops.convert_to_tensor(x, name="x") if isinstance(keep_prob, float) and not 0 < keep_prob <= 1: raise ValueError("keep_prob must be a scalar tensor or a float in the " "range (0, 1], got %g" % keep_prob) keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name="keep_prob") keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar()) noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x) # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype) # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) ret = x * math_ops.inv(keep_prob) * binary_tensor ret.set_shape(x.get_shape()) return ret
def beates_dropout(input_vals, keep_prob = 0.5): # Dropout keep_prob = ops.convert_to_tensor(keep_prob, dtype=input_vals.dtype, name="keep_prob") noise_shape = array_ops.shape(input_vals) # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, dtype=input_vals.dtype) # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) ret = input_vals * binary_tensor ret.set_shape(input_vals.get_shape()) return ret
def testStudentTWithAbsDfSoftplusScale(self): with self.test_session(): df = constant_op.constant([-3.2, -4.6]) mu = constant_op.constant([-4.2, 3.4]) sigma = constant_op.constant([-6.4, -8.8]) student = student_t.StudentTWithAbsDfSoftplusScale(df=df, loc=mu, scale=sigma) self.assertAllClose( math_ops.floor(math_ops.abs(df)).eval(), student.df.eval()) self.assertAllClose(mu.eval(), student.loc.eval()) self.assertAllClose( nn_ops.softplus(sigma).eval(), student.scale.eval())
def cosine_decay( learn_rate, # learning rate epoch, # epoch batch, # batch epoch decay, # decay decay_min_fraction, alpha, # alpha epochs, final_epochs, # finalepoch delay=0, name=None, ): with ops.name_scope(name, "LR_Finetune", [learn_rate, epoch]): # learning_rate = ops.convert_to_tensor( # learning_rate, name="initial_learning_rate") learn_rate = ops.convert_to_tensor(learn_rate, name="initial_learning_rate") dtype = tf.float32 learn_rate = math_ops.cast(learn_rate, dtype) batch = math_ops.cast(batch, dtype) final_epochs = math_ops.cast(final_epochs, dtype) alpha = math_ops.cast(alpha, dtype) decay = math_ops.cast(decay, dtype) epoch = math_ops.cast(epoch, dtype) completed_fraction = (epoch - delay) / batch lam = control_flow_ops.cond( math_ops.less_equal(epoch, delay), lambda: learn_rate, lambda: learn_rate * (decay**math_ops.floor(completed_fraction)), lambda: learn_rate * tf.math.maximum((decay**math_ops.floor( completed_fraction)), decay_min_fraction), ) return control_flow_ops.cond( math_ops.less_equal(epoch, epochs - final_epochs), lambda: lam, lambda: lam * (alpha + (1 - alpha) * (0.5 + 0.5 * math_ops.cos( (epoch - epochs + final_epochs) / final_epochs * 3.14159))), )
def __call__(self, step): with ops.name_scope_v2(self.name or "ExponentialDecay") as name: initial_learning_rate = ops.convert_to_tensor_v2_with_dispatch( self.initial_learning_rate, name="initial_learning_rate") dtype = initial_learning_rate.dtype decay_steps = math_ops.cast(self.decay_steps, dtype) decay_rate = math_ops.cast(self.decay_rate, dtype) global_step_recomp = math_ops.cast(step, dtype) p = global_step_recomp / decay_steps if self.staircase: p = math_ops.floor(p) return math_ops.multiply( initial_learning_rate, math_ops.pow(decay_rate, p), name=name)
def cdf(self, x, name="cdf"): """Cumulative density function. Args: x: Non-negative floating point tensor with dtype `dtype` and whose shape can be broadcast with `self.lam`. name: A name for this operation. Returns: The CDF of the events. """ with ops.name_scope(self.name): with ops.name_scope(name, values=[self.lam, x]): x = self._check_x(x, check_integer=False) return math_ops.igammac(math_ops.floor(x + 1), self.lam)
def __init__(self, df, validate_args=False, allow_nan_stats=True, name="Chi2WithAbsDf"): parameters = locals() with ops.name_scope(name, values=[df]): super(Chi2WithAbsDf, self).__init__(df=math_ops.floor(math_ops.abs(df, name="abs_df"), name="floor_abs_df"), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=name) self._parameters = parameters
def augm(t, arg): t = tf.image.random_contrast(t, lower=1 - arg.contrast_var, upper=1 + arg.contrast_var) t = tf.image.random_brightness(t, arg.brightness_var) t = tf.image.random_saturation(t, 1 - arg.saturation_var, 1 + arg.saturation_var) t = tf.image.random_hue(t, max_delta=arg.hue_var) random_tensor = 1. - arg.p_flip + random_ops.random_uniform(shape=[1], dtype=t.dtype) binary_tensor = math_ops.floor(random_tensor) augmented = binary_tensor * t + (1 - binary_tensor) * (1 - t) return augmented
def __call__(self, step): with ops.name_scope_v2(self.name or "InverseTimeDecay") as name: initial_learning_rate = ops.convert_to_tensor_v2_with_dispatch( self.initial_learning_rate, name="initial_learning_rate") dtype = initial_learning_rate.dtype decay_steps = math_ops.cast(self.decay_steps, dtype) decay_rate = math_ops.cast(self.decay_rate, dtype) global_step_recomp = math_ops.cast(step, dtype) p = global_step_recomp / decay_steps if self.staircase: p = math_ops.floor(p) const = math_ops.cast(constant_op.constant(1), dtype) denom = math_ops.add(const, math_ops.multiply(decay_rate, p)) return math_ops.divide(initial_learning_rate, denom, name=name)
def __init__(self, df, mu, sigma, validate_args=False, allow_nan_stats=True, name="StudentTWithAbsDfSoftplusSigma"): with ops.name_scope(name, values=[df, mu, sigma]) as ns: super(StudentTWithAbsDfSoftplusSigma, self).__init__(df=math_ops.floor(math_ops.abs(df)), mu=mu, sigma=nn.softplus(sigma), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns)
def dropped_inputs_training(): with ops.name_scope("drop_activation_training"): x = ops.convert_to_tensor(inputs, name="x") if not x.dtype.is_floating: raise ValueError( "x has to be a floating point tensor since it's going to" " be scaled. Got a %s tensor instead." % x.dtype) if isinstance(self.p, numbers.Real) and not 0 < self.p <= 1: raise ValueError( "p must be a scalar tensor or a float in the " "range (0, 1], got %g" % self.p) # Early return is nothin to be dropped if isinstance(self.p, float) and self.p == 1.: return nn.relu(x) if context.executing_eagerly(): if isinstance(self.p, ops.EagerTensor): if self.p.numpy() == 1: return nn.relu(x) else: p = ops.convert_to_tensor(self.p, dtype=x.dtype, name="p") p.get_shape().assert_is_compatible_with( tensor_shape.scalar()) # Do nothing if we know keep_prob == 1 if tensor_util.constant_value(p) == 1: return nn.relu(x) noise_shape = array_ops.shape(x) random_tensor = 1 - p random_tensor += random_ops.random_uniform(noise_shape, seed=self.seed, dtype=x.dtype) # random_tensor ~ uniform distrib [1 - p, 2 - p), ex: [0.05, 1.05) binary_tensor = math_ops.floor(random_tensor) # in binary tensor ~ 5% of are set 1 , 95% are set 0 # drop 95% of the negative part, keep all in the positive part # old implementation: # ret = - binary_tensor*nn.relu((-x)) + nn.relu(x) # new implementation, only 1 relu operation ret = binary_tensor * x + (1 - binary_tensor) * nn.relu(x) if not context.executing_eagerly(): ret.set_shape(x.get_shape()) return ret
def call(self, inputs, state): """Firing rate model RNN: new_gate_inputs = dt_over_tau*(W * input + U * state + B)+(1 - dt_over_tau)*gate_inputs. output = new_state = act(new_gate_inputs) """ #apply dropout keep_prob = random_ops.random_uniform(array_ops.shape(state)) + kp keep_prob = math_ops.floor(keep_prob) state = keep_prob * state #one timestep calculation state = nn_ops.bias_add( math_ops.matmul( array_ops.concat([inputs, self._activation(state)], 1), self._kernel), self._bias) * 0.1 + 0.9 * state return state, state
def spatial_drop_noise2(net, p, kernel): input_shape = net.get_shape().as_list() noise_shape = np.array([input_shape[0], input_shape[1], input_shape[2], 1], dtype=np.int64) random_tensor = 1. - p random_tensor += tf.random_uniform(noise_shape) binary_tensor = math_ops.floor(random_tensor) binary_tensor = slim.max_pool2d(binary_tensor, kernel, stride=1, padding='SAME') noise_tensor = tf.random_normal(net.get_shape()) * binary_tensor binary_tensor = tf.ones_like(binary_tensor) - binary_tensor ret = net * binary_tensor ret.set_shape(net.get_shape()) return ret + noise_tensor, binary_tensor
def apply_dropout(): keep_prob = 1 - rate # uniform [keep_prob, 1.0 + keep_prob) random_tensor = keep_prob random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype) # 0. if [keep_prob, 1.0) and 1. if [1.0, 1.0 + keep_prob) binary_tensor = math_ops.floor(random_tensor) # save binary tensor to variable assign_op = binary_tensor_var.assign(binary_tensor) with tf.control_dependencies([assign_op]): ret = math_ops.divide(x, keep_prob) * binary_tensor #ret = tf.Print(ret,["apply dropout and save", _is_reuse_binary_tensor_var]) return ret
def clr(base_lr, max_lr, step_size, clr_iterations, name=None): if clr_iterations is None: raise ValueError('global_step is required for clr!') with ops.name_scope(name, 'clr', [base_lr, max_lr, step_size, clr_iterations]) as name: base_lr = ops.convert_to_tensor(base_lr, name="learning_rate") dtype = base_lr.dtype max_lr = math_ops.cast(max_lr, dtype) step_size = math_ops.cast(step_size, dtype) clr_iterations = math_ops.cast(clr_iterations, dtype) cycle = math_ops.floor(1 + clr_iterations / (2 * step_size)) x = math_ops.abs(clr_iterations / step_size - 2 * cycle + 1) return math_ops.add( base_lr, math_ops.mul(math_ops.subtract(max_lr, base_lr), math_ops.maximum(0., math_ops.subtract(1., x))))
def mode(self, name="mode"): """Mode of the distribution. Note that when `lam` is an integer, there are actually two modes. Namely, `lam` and `lam - 1` are both modes. Here we return only the larger of the two modes. Args: name: Name for the op. Returns: mode: `Tensor` of the same type and shape as `lam`. """ with ops.name_scope(self.name): with ops.name_scope(name, values=[self.lam]): return math_ops.floor(self.lam)
def mode(self, name="mode"): """Mode of the distribution. Note that when `(n + 1) * p` is an integer, there are actually two modes. Namely, `(n + 1) * p` and `(n + 1) * p - 1` are both modes. Here we return only the larger of the two modes. Args: name: The name for this op. Returns: The mode of the Binomial distribution. """ with ops.name_scope(self.name): with ops.op_scope([self._n, self._p], name): return math_ops.floor((self._n + 1) * self._p)
def testFloorDivide(self): x = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype( np.float32).reshape([1, 3, 2]) y = (1 + np.linspace(0, 5, np.prod([1, 3, 2]))).astype( np.float32).reshape([1, 3, 2]) np_out = np.floor_divide(x, y + 0.1) with self.session(use_gpu=True) as sess: inx = ops.convert_to_tensor(x) iny = ops.convert_to_tensor(y + 0.1) ofunc = inx / iny out_func2 = math_ops.floor(ofunc) tf_out = self.evaluate(out_func2) self.assertAllClose(np_out, tf_out)
def _sample_n(self, n, seed=None): # Uniform variates must be sampled from the open-interval `(0, 1)` rather # than `[0, 1)`. To do so, we use `np.finfo(self.dtype.as_numpy_dtype).tiny` # because it is the smallest, positive, "normal" number. A "normal" number # is such that the mantissa has an implicit leading 1. Normal, positive # numbers x, y have the reasonable property that, `x + y >= max(x, y)`. In # this case, a subnormal number (i.e., np.nextafter) can cause us to sample # 0. sampled = random_ops.random_uniform( array_ops.concat([[n], array_ops.shape(self._probs)], 0), minval=np.finfo(self.dtype.as_numpy_dtype).tiny, maxval=1., seed=seed, dtype=self.dtype) return math_ops.floor( math_ops.log(sampled) / math_ops.log1p(-self.probs))
def __init__(self, df, loc, scale, validate_args=False, allow_nan_stats=True, name="StudentTWithAbsDfSoftplusScale"): parameters = locals() with ops.name_scope(name, values=[df, scale]) as ns: super(StudentTWithAbsDfSoftplusScale, self).__init__( df=math_ops.floor(math_ops.abs(df)), loc=loc, scale=nn.softplus(scale, name="softplus_scale"), validate_args=validate_args, allow_nan_stats=allow_nan_stats, name=ns) self._parameters = parameters
def _histogram(values, value_range, nbins=100, dtype=np.int32, name=None): """Return histogram of values. Given the tensor `values`, this operation returns a rank 1 histogram counting the number of entries in `values` that fell into every bin. The bins are equal width and determined by the arguments `value_range` and `nbins`. Args: values: Numeric `Tensor`. value_range: Shape [2] `Tensor` of same `dtype` as `values`. values <= value_range[0] will be mapped to hist[0], values >= value_range[1] will be mapped to hist[-1]. nbins: Scalar `int32 Tensor`. Number of histogram bins. dtype: dtype for returned histogram. name: A name for this operation (defaults to 'histogram'). Returns: A 1-D `Tensor` holding histogram of values. """ with ops.name_scope(name, 'histogram', [values, value_range, nbins]) as scope: values = ops.convert_to_tensor(values, name='values') values = gen_array_ops.reshape(values, [-1]) value_range = ops.convert_to_tensor(value_range, name='value_range') nbins = ops.convert_to_tensor(nbins, dtype=np.int32, name='nbins') nbins_float = math_ops.cast(nbins, values.dtype) # Map tensor values that fall within value_range to [0, 1]. scaled_values = math_ops.truediv(values - value_range[0], value_range[1] - value_range[0], name='scaled_values') # map tensor values within the open interval value_range to {0,.., nbins-1}, # values outside the open interval will be zero or less, or nbins or more. indices = math_ops.floor(nbins_float * scaled_values, name='indices') # Clip edge cases (e.g. value = value_range[1]) or "outliers." indices = math_ops.cast( clip_ops.clip_by_value(indices, 0, nbins_float - 1), np.int32) return math_ops.unsorted_segment_sum(array_ops.ones_like(indices, dtype=dtype), indices, nbins, name=scope)