def kl_multivariate_normal(loc_one, scale_one, loc_two=0.0, scale_two=1.0):
"""Calculate the KL of multivariate normal distributions with
diagonal covariances.
Parameters
----------
loc_one : tf.Tensor
A 0-D tensor, 1-D tensor of length n, or 2-D tensor of shape M
x n where each row represents the mean of a n-dimensional
Gaussian.
scale_one : tf.Tensor
A tensor of same shape as ``loc_one``, representing the
standard deviation.
loc_two : tf.Tensor, optional
A tensor of same shape as ``loc_one``, representing the
mean of another Gaussian.
scale_two : tf.Tensor, optional
A tensor of same shape as ``loc_one``, representing the
standard deviation of another Gaussian.
Returns
-------
tf.Tensor
For 0-D or 1-D tensor inputs, outputs the 0-D tensor
``KL( N(z; loc_one, scale_one) || N(z; loc_two, scale_two) )``
For 2-D tensor inputs, outputs the 1-D tensor
``[KL( N(z; loc_one[m,:], scale_one[m,:]) || N(z; loc_two[m,:], scale_two[m,:]) )]_{m=1}^M``
Raises
------
InvalidArgumentError
If the location variables have Inf or NaN values, or if the scale
variables are not positive.
"""
dependencies = [tf.verify_tensor_all_finite(loc_one, msg=''),
tf.verify_tensor_all_finite(loc_two, msg=''),
tf.assert_positive(scale_one),
tf.assert_positive(scale_two)]
loc_one = control_flow_ops.with_dependencies(dependencies, loc_one)
scale_one = control_flow_ops.with_dependencies(dependencies, scale_one)
loc_one = tf.cast(loc_one, tf.float32)
scale_one = tf.cast(scale_one, tf.float32)
if loc_two == 0.0 and scale_two == 1.0:
# With default arguments, we can avoid some intermediate computation.
out = tf.square(scale_one) + tf.square(loc_one) - \
1.0 - 2.0 * tf.log(scale_one)
else:
loc_two = control_flow_ops.with_dependencies(dependencies, loc_two)
scale_two = control_flow_ops.with_dependencies(dependencies, scale_two)
loc_two = tf.cast(loc_two, tf.float32)
scale_two = tf.cast(scale_two, tf.float32)
out = tf.square(scale_one/scale_two) + \
tf.square((loc_two - loc_one)/scale_two) - \
1.0 + 2.0 * tf.log(scale_two) - 2.0 * tf.log(scale_one)
if len(out.get_shape()) <= 1: # scalar or vector
return 0.5 * tf.reduce_sum(out)
else: # matrix
return 0.5 * tf.reduce_sum(out, 1)
def __init__(self,
concentration,
rate,
validate_args=False,
allow_nan_stats=True,
name="InverseGamma"):
"""Construct InverseGamma with `concentration` and `rate` parameters.
The parameters `concentration` and `rate` must be shaped in a way that
supports broadcasting (e.g. `concentration + rate` is a valid operation).
Args:
concentration: Floating point tensor, the concentration params of the
distribution(s). Must contain only positive values.
rate: Floating point tensor, the inverse scale params of the
distribution(s). Must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
TypeError: if `concentration` and `rate` are different dtypes.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[concentration, rate]) as name:
dtype = dtype_util.common_dtype([concentration, rate],
preferred_dtype=tf.float32)
concentration = tf.convert_to_tensor(
concentration, name="concentration", dtype=dtype)
rate = tf.convert_to_tensor(rate, name="rate", dtype=dtype)
with tf.control_dependencies([
tf.assert_positive(
concentration,
message="Concentration must be positive."),
tf.assert_positive(
rate,
message="Rate must be positive."),
] if validate_args else []):
self._concentration = tf.identity(concentration, name="concentration")
self._rate = tf.identity(rate, name="rate")
tf.assert_same_float_dtype([self._concentration, self._rate])
super(InverseGamma, self).__init__(
dtype=self._concentration.dtype,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
reparameterization_type=reparameterization.FULLY_REPARAMETERIZED,
parameters=parameters,
graph_parents=[self._concentration, self._rate],
name=name)
def rbf(X, X2=None, lengthscale=1.0, variance=1.0):
"""Radial basis function kernel, also known as the squared
exponential or exponentiated quadratic. It is defined as
$k(x, x') = \sigma^2 \exp\Big(
-\\frac{1}{2} \sum_{d=1}^D \\frac{1}{\ell_d^2} (x_d - x'_d)^2 \Big)$
for output variance $\sigma^2$ and lengthscale $\ell^2$.
The kernel is evaluated over all pairs of rows, `k(X[i, ], X2[j, ])`.
If `X2` is not specified, then it evaluates over all pairs
of rows in `X`, `k(X[i, ], X[j, ])`. The output is a matrix
where each entry (i, j) is the kernel over the ith and jth rows.
Args:
X: tf.Tensor.
N x D matrix of N data points each with D features.
X2: tf.Tensor.
N x D matrix of N data points each with D features.
lengthscale: tf.Tensor.
Lengthscale parameter, a positive scalar or D-dimensional vector.
variance: tf.Tensor.
Output variance parameter, a positive scalar.
#### Examples
```python
X = tf.random_normal([100, 5])
K = ed.rbf(X)
assert K.shape == (100, 100)
```
"""
lengthscale = tf.convert_to_tensor(lengthscale)
variance = tf.convert_to_tensor(variance)
dependencies = [tf.assert_positive(lengthscale),
tf.assert_positive(variance)]
lengthscale = control_flow_ops.with_dependencies(dependencies, lengthscale)
variance = control_flow_ops.with_dependencies(dependencies, variance)
X = tf.convert_to_tensor(X)
X = X / lengthscale
Xs = tf.reduce_sum(tf.square(X), 1)
if X2 is None:
X2 = X
X2s = Xs
else:
X2 = tf.convert_to_tensor(X2)
X2 = X2 / lengthscale
X2s = tf.reduce_sum(tf.square(X2), 1)
square = tf.reshape(Xs, [-1, 1]) + tf.reshape(X2s, [1, -1]) - \
2 * tf.matmul(X, X2, transpose_b=True)
output = variance * tf.exp(-square / 2)
return output
def _validate(self):
vops = [tf.assert_positive(self._scale),
tf.assert_positive(self._high - self._low),
tf.verify_tensor_all_finite(self._high,
"Upper bound not finite"),
tf.verify_tensor_all_finite(self._low,
"Lower bound not finite"),
tf.verify_tensor_all_finite(self._loc,
"Loc not finite"),
tf.verify_tensor_all_finite(self._scale,
"Scale not finite"),
]
return tf.group(*vops, name="ValidationOps")
def kl_multivariate_normal(loc_one, scale_one, loc_two=0.0, scale_two=1.0):
"""Calculate the KL of multivariate normal distributions with
diagonal covariances.
Parameters
----------
loc_one : tf.Tensor
n-dimensional vector, or M x n-dimensional matrix where each
row represents the mean of a n-dimensional Gaussian
scale_one : tf.Tensor
n-dimensional vector, or M x n-dimensional matrix where each
row represents the standard deviation of a n-dimensional Gaussian
loc_two : tf.Tensor, optional
n-dimensional vector, or M x n-dimensional matrix where each
row represents the mean of a n-dimensional Gaussian
scale_two : tf.Tensor, optional
n-dimensional vector, or M x n-dimensional matrix where each
row represents the standard deviation of a n-dimensional Gaussian
Returns
-------
tf.Tensor
for scalar or vector inputs, outputs the scalar
``KL( N(z; loc_one, scale_one) || N(z; loc_two, scale_two) )``
for matrix inputs, outputs the vector
``[KL( N(z; loc_one[m,:], scale_one[m,:]) || N(z; loc_two[m,:], scale_two[m,:]) )]_{m=1}^M``
Raises
------
InvalidArgumentError
If the location variables have Inf or NaN values, or if the scale
variables are not positive.
"""
dependencies = [tf.verify_tensor_all_finite(loc_one, msg=''),
tf.verify_tensor_all_finite(loc_two, msg=''),
tf.assert_positive(scale_one),
tf.assert_positive(scale_two)]
loc_one = control_flow_ops.with_dependencies(dependencies, loc_one)
loc_two = control_flow_ops.with_dependencies(dependencies, loc_two)
scale_one = control_flow_ops.with_dependencies(dependencies, scale_one)
scale_two = control_flow_ops.with_dependencies(dependencies, scale_two)
if loc_two == 0.0 and scale_two == 1.0:
return 0.5 * tf.reduce_sum(
tf.square(scale_one) + tf.square(loc_one) - \
1.0 - 2.0 * tf.log(scale_one))
else:
return 0.5 * tf.reduce_sum(
tf.square(scale_one/scale_two) + \
tf.square((loc_two - loc_one)/scale_two) - \
1.0 + 2.0 * tf.log(scale_two) - 2.0 * tf.log(scale_one), 1)
def _maybe_assert_valid_sample(self, x):
tf.assert_same_float_dtype(tensors=[x], dtype=self.dtype)
if not self.validate_args:
return x
return control_flow_ops.with_dependencies([
tf.assert_positive(x),
], x)
def calculate_reshape(original_shape, new_shape, validate=False, name=None):
"""Calculates the reshaped dimensions (replacing up to one -1 in reshape)."""
batch_shape_static = tensor_util.constant_value_as_shape(new_shape)
if batch_shape_static.is_fully_defined():
return np.int32(batch_shape_static.as_list()), batch_shape_static, []
with tf.name_scope(name, "calculate_reshape", [original_shape, new_shape]):
original_size = tf.reduce_prod(original_shape)
implicit_dim = tf.equal(new_shape, -1)
size_implicit_dim = (
original_size // tf.maximum(1, -tf.reduce_prod(new_shape)))
new_ndims = tf.shape(new_shape)
expanded_new_shape = tf.where( # Assumes exactly one `-1`.
implicit_dim, tf.fill(new_ndims, size_implicit_dim), new_shape)
validations = [] if not validate else [
tf.assert_rank(
original_shape, 1, message="Original shape must be a vector."),
tf.assert_rank(new_shape, 1, message="New shape must be a vector."),
tf.assert_less_equal(
tf.count_nonzero(implicit_dim, dtype=tf.int32),
1,
message="At most one dimension can be unknown."),
tf.assert_positive(
expanded_new_shape, message="Shape elements must be >=-1."),
tf.assert_equal(
tf.reduce_prod(expanded_new_shape),
original_size,
message="Shape sizes do not match."),
]
return expanded_new_shape, batch_shape_static, validations
def _maybe_assert_valid_x(self, x):
if not self.validate_args:
return x
is_valid = tf.assert_positive(
x[..., 1:] - x[..., :-1],
message="Forward transformation input must be strictly increasing.")
return control_flow_ops.with_dependencies([is_valid], x)
def disable_some_fgs():
# We want to delete a randomly-selected subset of fg_inds of
# size `fg_inds.shape[0] - max_fg`.
# We shuffle along the dimension 0 and then we get the first
# num_fg_inds - max_fg indices and we disable them.
shuffled_inds = tf.random_shuffle(fg_inds, seed=self._seed)
disable_place = (tf.shape(fg_inds)[0] - max_fg)
# This function should never run if num_fg_inds <= max_fg, so we
# add an assertion to catch the wrong behaviour if it happens.
integrity_assertion = tf.assert_positive(
disable_place,
message="disable_place in disable_some_fgs is negative."
)
with tf.control_dependencies([integrity_assertion]):
disable_inds = shuffled_inds[:disable_place]
is_disabled = tf.sparse_to_dense(
sparse_indices=disable_inds,
sparse_values=True, default_value=False,
output_shape=tf.cast(proposals_label_shape, tf.int64),
# We are shuffling the indices, so they may not be ordered.
validate_indices=False
)
return tf.where(
condition=is_disabled,
# We set it to -label for debugging purposes.
x=tf.negative(proposals_label),
y=proposals_label
)
def enqueuer(raw_data, batch_size, num_steps, name=None):
"""Iterate on the raw PTB data.
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from ptb_raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "InputEnqueuer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data, name = "raw_data", dtype = tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0 : batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(epoch_size, message = "epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name = "epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue() # output queue index from the queue object (relies on tf.train.Supervisor)
x = tf.slice(data, [0, i * num_steps], [batch_size, num_steps]) # slice data
y = tf.slice(data, [0, i * num_steps + 1], [batch_size, num_steps])
return x, y
def check_3d_image(image, require_static=True):
"""Assert that we are working with properly shaped image.
Args:
image: 3-D Tensor of shape [height, width, channels]
require_static: If `True`, requires that all dimensions of `image` are
known and non-zero.
Raises:
ValueError: if `image.shape` is not a 3-vector.
Returns:
An empty list, if `image` has fully defined dimensions. Otherwise, a list
containing an assert op is returned.
"""
try:
image_shape = image.get_shape().with_rank(3)
except ValueError:
raise ValueError("'image' must be three-dimensional.")
if require_static and not image_shape.is_fully_defined():
raise ValueError("'image' must be fully defined.")
if any(x == 0 for x in image_shape):
raise ValueError("all dims of 'image.shape' must be > 0: %s" %
image_shape)
if not image_shape.is_fully_defined():
return [tf.assert_positive(tf.shape(image),
["all dims of 'image.shape' "
"must be > 0."])]
else:
return []
def test_raises_when_negative(self):
with self.test_session():
freddie = tf.constant([-1, -2], name="freddie")
with tf.control_dependencies([tf.assert_positive(freddie)]):
out = tf.identity(freddie)
with self.assertRaisesOpError("freddie"):
out.eval()
def __init__(self,
loc=0.,
scale=1.,
validate_args=False,
name="gumbel"):
"""Instantiates the `Gumbel` bijector.
Args:
loc: Float-like `Tensor` that is the same dtype and is
broadcastable with `scale`.
This is `loc` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
scale: Positive Float-like `Tensor` that is the same dtype and is
broadcastable with `loc`.
This is `scale` in `Y = g(X) = exp(-exp(-(X - loc) / scale))`.
validate_args: Python `bool` indicating whether arguments should be
checked for correctness.
name: Python `str` name given to ops managed by this object.
"""
self._graph_parents = []
self._name = name
self._validate_args = validate_args
with self._name_scope("init", values=[loc, scale]):
self._loc = tf.convert_to_tensor(loc, name="loc")
self._scale = tf.convert_to_tensor(scale, name="scale")
tf.assert_same_float_dtype([self._loc, self._scale])
if validate_args:
self._scale = control_flow_ops.with_dependencies([
tf.assert_positive(
self._scale, message="Argument scale was not positive")
], self._scale)
super(Gumbel, self).__init__(
validate_args=validate_args,
forward_min_event_ndims=0,
name=name)
def ptb_producer(raw_data, batch_size, num_steps, name=None):
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
dtype=tf.int32, name="raw_data")
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0: batch_len*batch_size],
[batch_size, batch_len])
epoch_size = (batch_len-1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="batch size too large")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i*num_steps],
[batch_size, (i+1)*num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i*num_steps+1],
[batch_size, (i+1)*num_steps+1])
y.set_shape([batch_size, num_steps])
return x, y
def test_raises_when_zero(self):
with self.test_session():
meechum = tf.constant([0], name="meechum")
with tf.control_dependencies([tf.assert_positive(meechum)]):
out = tf.identity(meechum)
with self.assertRaisesOpError("meechum"):
out.eval()
def reduce_mean(seq_batch, allow_empty=False):
"""Compute the mean of each sequence in a SequenceBatch.
Args:
seq_batch (SequenceBatch): a SequenceBatch with the following attributes:
values (Tensor): a Tensor of shape (batch_size, seq_length, :, ..., :)
mask (Tensor): if the mask values are arbitrary floats (rather than binary), the mean will be
a weighted average.
allow_empty (bool): allow computing the average of an empty sequence. In this case, we assume 0/0 == 0, rather
than NaN. Default is False, causing an error to be thrown.
Returns:
Tensor: of shape (batch_size, :, ..., :)
"""
values, mask = seq_batch.values, seq_batch.mask
# compute weights for the average
sums = tf.reduce_sum(mask, 1, keep_dims=True) # (batch_size, 1)
if allow_empty:
asserts = [] # no assertion
sums = tf.select(tf.equal(sums, 0), tf.ones(tf.shape(sums)), sums) # replace 0's with 1's
else:
asserts = [tf.assert_positive(sums)] # throw error if 0's exist
with tf.control_dependencies(asserts):
weights = mask / sums # (batch_size, seq_length)
return weighted_sum(seq_batch, weights)
def __init__(self,
scale,
validate_args=False,
allow_nan_stats=True,
name="HalfNormal"):
"""Construct HalfNormals with scale `scale`.
Args:
scale: Floating point tensor; the scales of the distribution(s).
Must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[scale]) as name:
with tf.control_dependencies([tf.assert_positive(scale)]
if validate_args else []):
self._scale = tf.identity(scale, name="scale")
super(HalfNormal, self).__init__(
dtype=self._scale.dtype,
reparameterization_type=tf.distributions.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._scale],
name=name)
def __init__(self,
total_count,
logits=None,
probs=None,
validate_args=False,
allow_nan_stats=True,
name="NegativeBinomial"):
"""Construct NegativeBinomial distributions.
Args:
total_count: Non-negative floating-point `Tensor` with shape
broadcastable to `[B1,..., Bb]` with `b >= 0` and the same dtype as
`probs` or `logits`. Defines this as a batch of `N1 x ... x Nm`
different Negative Binomial distributions. In practice, this represents
the number of negative Bernoulli trials to stop at (the `total_count`
of failures), but this is still a valid distribution when
`total_count` is a non-integer.
logits: Floating-point `Tensor` with shape broadcastable to
`[B1, ..., Bb]` where `b >= 0` indicates the number of batch dimensions.
Each entry represents logits for the probability of success for
independent Negative Binomial distributions and must be in the open
interval `(-inf, inf)`. Only one of `logits` or `probs` should be
specified.
probs: Positive floating-point `Tensor` with shape broadcastable to
`[B1, ..., Bb]` where `b >= 0` indicates the number of batch dimensions.
Each entry represents the probability of success for independent
Negative Binomial distributions and must be in the open interval
`(0, 1)`. Only one of `logits` or `probs` should be specified.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[total_count, logits, probs]) as name:
dtype = dtype_util.common_dtype([total_count, logits, probs],
preferred_dtype=tf.float32)
self._logits, self._probs = distribution_util.get_logits_and_probs(
logits, probs, validate_args=validate_args, name=name, dtype=dtype)
total_count = tf.convert_to_tensor(
total_count, name="total_count", dtype=dtype)
with tf.control_dependencies([tf.assert_positive(total_count)]
if validate_args else []):
self._total_count = tf.identity(total_count, name="total_count")
super(NegativeBinomial, self).__init__(
dtype=self._probs.dtype,
reparameterization_type=reparameterization.NOT_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._total_count, self._probs, self._logits],
name=name)
def _maybe_assert_valid_concentration(self, concentration, validate_args):
"""Checks the validity of a concentration parameter."""
if not validate_args:
return concentration
return control_flow_ops.with_dependencies([
tf.assert_positive(
concentration, message="Concentration parameter must be positive."),
], concentration)
def multivariate_rbf(x, y=0.0, sigma=1.0, l=1.0):
"""Squared-exponential kernel
.. math:: k(x, y) = \sigma^2 \exp{ -1/(2l^2) \sum_i (x_i - y_i)^2 }
Parameters
----------
x : tf.Tensor
A n-D tensor.
y : tf.Tensor, optional
A tensor of same shape as ``x``.
sigma : tf.Tensor, optional
A 0-D tensor, representing the standard deviation of radial
basis function.
l : tf.Tensor, optional
A 0-D tensor, representing the lengthscale of radial basis
function.
Returns
-------
tf.Tensor
A tensor of one less dimension than the input.
Raises
------
InvalidArgumentError
If the mean variables have Inf or NaN values, or if the scale
and length variables are not positive.
"""
dependencies = [tf.verify_tensor_all_finite(x, msg=''),
tf.verify_tensor_all_finite(y, msg=''),
tf.assert_positive(sigma),
tf.assert_positive(l)]
x = control_flow_ops.with_dependencies(dependencies, x)
y = control_flow_ops.with_dependencies(dependencies, y)
sigma = control_flow_ops.with_dependencies(dependencies, sigma)
l = control_flow_ops.with_dependencies(dependencies, l)
x = tf.cast(x, dtype=tf.float32)
y = tf.cast(y, dtype=tf.float32)
sigma = tf.cast(sigma, dtype=tf.float32)
l = tf.cast(l, dtype=tf.float32)
return tf.pow(sigma, 2.0) * \
tf.exp(-1.0/(2.0*tf.pow(l, 2.0)) * \
tf.reduce_sum(tf.pow(x - y , 2.0)))
def __init__(self,
concentration,
scale=1.,
validate_args=False,
allow_nan_stats=True,
name="Pareto"):
"""Construct Pareto distribution with `concentration` and `scale`.
Args:
concentration: Floating point tensor. Must contain only positive values.
scale: Floating point tensor, equivalent to `mode`. `scale` also
restricts the domain of this distribution to be in `[scale, inf)`.
Must contain only positive values. Default value: `1`.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs. Default value: `False` (i.e. do not validate args).
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
Default value: `True`.
name: Python `str` name prefixed to Ops created by this class.
Default value: 'Pareto'.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[concentration, scale]):
dtype = dtype_util.common_dtype([concentration, scale], tf.float32)
self._concentration = tf.convert_to_tensor(
concentration, name="concentration", dtype=dtype)
self._scale = tf.convert_to_tensor(scale, name="scale", dtype=dtype)
with tf.control_dependencies([
tf.assert_positive(self._concentration),
tf.assert_positive(self._scale)] if validate_args else []):
self._concentration = tf.identity(
self._concentration, name="concentration")
self._scale = tf.identity(self._scale, name="scale")
super(Pareto, self).__init__(
dtype=self._concentration.dtype,
reparameterization_type=reparameterization.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._concentration, self._scale],
name=name)
def __init__(self,
df,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="StudentT"):
"""Construct Student's t distributions.
The distributions have degree of freedom `df`, mean `loc`, and scale
`scale`.
The parameters `df`, `loc`, and `scale` must be shaped in a way that
supports broadcasting (e.g. `df + loc + scale` is a valid operation).
Args:
df: Floating-point `Tensor`. The degrees of freedom of the
distribution(s). `df` must contain only positive values.
loc: Floating-point `Tensor`. The mean(s) of the distribution(s).
scale: Floating-point `Tensor`. The scaling factor(s) for the
distribution(s). Note that `scale` is not technically the standard
deviation of this distribution but has semantics more similar to
standard deviation than variance.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
TypeError: if loc and scale are different dtypes.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[df, loc, scale]) as name:
dtype = dtype_util.common_dtype([df, loc, scale], tf.float32)
df = tf.convert_to_tensor(df, name="df", dtype=dtype)
loc = tf.convert_to_tensor(loc, name="loc", dtype=dtype)
scale = tf.convert_to_tensor(scale, name="scale", dtype=dtype)
with tf.control_dependencies([tf.assert_positive(df)]
if validate_args else []):
self._df = tf.identity(df)
self._loc = tf.identity(loc)
self._scale = tf.identity(scale)
tf.assert_same_float_dtype(
(self._df, self._loc, self._scale))
super(StudentT, self).__init__(
dtype=self._scale.dtype,
reparameterization_type=reparameterization.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._df, self._loc, self._scale],
name=name)
def __init__(self,
rate=None,
log_rate=None,
validate_args=False,
allow_nan_stats=True,
name="Poisson"):
"""Initialize a batch of Poisson distributions.
Args:
rate: Floating point tensor, the rate parameter. `rate` must be positive.
Must specify exactly one of `rate` and `log_rate`.
log_rate: Floating point tensor, the log of the rate parameter.
Must specify exactly one of `rate` and `log_rate`.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
ValueError: if none or both of `rate`, `log_rate` are specified.
TypeError: if `rate` is not a float-type.
TypeError: if `log_rate` is not a float-type.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[rate]) as name:
if (rate is None) == (log_rate is None):
raise ValueError("Must specify exactly one of `rate` and `log_rate`.")
elif log_rate is None:
rate = tf.convert_to_tensor(rate, name="rate")
if not rate.dtype.is_floating:
raise TypeError("rate.dtype ({}) is a not a float-type.".format(
rate.dtype.name))
with tf.control_dependencies([tf.assert_positive(rate)]
if validate_args else []):
self._rate = tf.identity(rate, name="rate")
self._log_rate = tf.log(rate, name="log_rate")
else:
log_rate = tf.convert_to_tensor(log_rate, name="log_rate")
if not log_rate.dtype.is_floating:
raise TypeError("log_rate.dtype ({}) is a not a float-type.".format(
log_rate.dtype.name))
self._rate = tf.exp(log_rate, name="rate")
self._log_rate = tf.convert_to_tensor(log_rate, name="log_rate")
super(Poisson, self).__init__(
dtype=self._rate.dtype,
reparameterization_type=tf.distributions.NOT_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._rate],
name=name)
def test_empty_tensor_doesnt_raise(self):
# A tensor is positive when it satisfies:
# For every element x_i in x, x_i > 0
# and an empty tensor has no elements, so this is trivially satisfied.
# This is standard set theory.
with self.test_session():
empty = tf.constant([], name="empty")
with tf.control_dependencies([tf.assert_positive(empty)]):
out = tf.identity(empty)
out.eval()
def __init__(self,
temperature,
logits=None,
probs=None,
validate_args=False,
allow_nan_stats=True,
name="RelaxedBernoulli"):
"""Construct RelaxedBernoulli distributions.
Args:
temperature: An 0-D `Tensor`, representing the temperature
of a set of RelaxedBernoulli distributions. The temperature should be
positive.
logits: An N-D `Tensor` representing the log-odds
of a positive event. Each entry in the `Tensor` parametrizes
an independent RelaxedBernoulli distribution where the probability of an
event is sigmoid(logits). Only one of `logits` or `probs` should be
passed in.
probs: An N-D `Tensor` representing the probability of a positive event.
Each entry in the `Tensor` parameterizes an independent Bernoulli
distribution. Only one of `logits` or `probs` should be passed in.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
ValueError: If both `probs` and `logits` are passed, or if neither.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[logits, probs, temperature]) as name:
dtype = dtype_util.common_dtype([logits, probs, temperature], tf.float32)
self._temperature = tf.convert_to_tensor(
temperature, name="temperature", dtype=dtype)
if validate_args:
with tf.control_dependencies([tf.assert_positive(temperature)]):
self._temperature = tf.identity(self._temperature)
self._logits, self._probs = distribution_util.get_logits_and_probs(
logits=logits, probs=probs, validate_args=validate_args, dtype=dtype)
super(RelaxedBernoulli, self).__init__(
distribution=logistic.Logistic(
self._logits / self._temperature,
1. / self._temperature,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name + "/Logistic"),
bijector=sigmoid_bijector.Sigmoid(validate_args=validate_args),
validate_args=validate_args,
name=name)
self._parameters = parameters
def _maybe_assert_valid_sample(self, x):
"""Checks the validity of a sample."""
if not self.validate_args:
return x
return control_flow_ops.with_dependencies([
tf.assert_positive(x, message="sample must be positive"),
tf.assert_less(
x,
tf.ones([], self.dtype),
message="sample must be less than `1`."),
], x)
def _maybe_attach_assertion(x):
if not validate_args:
return x
if assert_positive:
return control_flow_ops.with_dependencies([
tf.assert_positive(x, message="diagonal part must be positive"),
], x)
return control_flow_ops.with_dependencies([
tf.assert_none_equal(
x, tf.zeros([], x.dtype), message="diagonal part must be non-zero")
], x)
def _maybe_assert_valid_sample(self, x):
"""Checks the validity of a sample."""
if not self.validate_args:
return x
return control_flow_ops.with_dependencies([
tf.assert_positive(x, message="samples must be positive"),
tf.assert_near(
tf.ones([], dtype=self.dtype),
tf.reduce_sum(x, -1),
message="sample last-dimension must sum to `1`"),
], x)
def __init__(self,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="Gumbel"):
"""Construct Gumbel distributions with location and scale `loc` and `scale`.
The parameters `loc` and `scale` must be shaped in a way that supports
broadcasting (e.g. `loc + scale` is a valid operation).
Args:
loc: Floating point tensor, the means of the distribution(s).
scale: Floating point tensor, the scales of the distribution(s).
scale must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
Default value: `False`.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
Default value: `True`.
name: Python `str` name prefixed to Ops created by this class.
Default value: `'Gumbel'`.
Raises:
TypeError: if loc and scale are different dtypes.
"""
with tf.name_scope(name, values=[loc, scale]) as name:
dtype = dtype_util.common_dtype([loc, scale], preferred_dtype=tf.float32)
loc = tf.convert_to_tensor(loc, name="loc", dtype=dtype)
scale = tf.convert_to_tensor(scale, name="scale", dtype=dtype)
with tf.control_dependencies([tf.assert_positive(scale)]
if validate_args else []):
loc = tf.identity(loc, name="loc")
scale = tf.identity(scale, name="scale")
tf.assert_same_float_dtype([loc, scale])
self._gumbel_bijector = gumbel_bijector.Gumbel(
loc=loc, scale=scale, validate_args=validate_args)
super(Gumbel, self).__init__(
distribution=uniform.Uniform(
low=tf.zeros([], dtype=loc.dtype),
high=tf.ones([], dtype=loc.dtype),
allow_nan_stats=allow_nan_stats),
# The Gumbel bijector encodes the quantile
# function as the forward, and hence needs to
# be inverted.
bijector=invert_bijector.Invert(self._gumbel_bijector),
batch_shape=distribution_util.get_broadcast_shape(loc, scale),
name=name)
def __init__(self,
scale=1.,
concentration=1.,
validate_args=False,
name="weibull"):
"""Instantiates the `Weibull` bijector.
Args:
scale: Positive Float-type `Tensor` that is the same dtype and is
broadcastable with `concentration`.
This is `l` in `Y = g(X) = 1 - exp((-x / l) ** k)`.
concentration: Positive Float-type `Tensor` that is the same dtype and is
broadcastable with `scale`.
This is `k` in `Y = g(X) = 1 - exp((-x / l) ** k)`.
validate_args: Python `bool` indicating whether arguments should be
checked for correctness.
name: Python `str` name given to ops managed by this object.
"""
self._graph_parents = []
self._name = name
self._validate_args = validate_args
with self._name_scope("init", values=[scale, concentration]):
self._scale = tf.convert_to_tensor(scale, name="scale")
self._concentration = tf.convert_to_tensor(
concentration, name="concentration")
tf.assert_same_float_dtype([self._scale, self._concentration])
if validate_args:
self._scale = control_flow_ops.with_dependencies([
tf.assert_positive(
self._scale, message="Argument scale was not positive")
], self._scale)
self._concentration = control_flow_ops.with_dependencies([
tf.assert_positive(
self._concentration,
message="Argument concentration was not positive")
], self._concentration)
super(Weibull, self).__init__(
forward_min_event_ndims=0,
validate_args=validate_args,
name=name)
def ptb_producer(raw_data, batch_size, num_steps, name=None):
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
input_data = []
target_data = []
weight_data = []
for single_data in raw_data:
input_data.append(single_data[0])
target_data.append(single_data[1])
weight_data.append(single_data[2])
#print(len(input_data))
input_data = tf.convert_to_tensor(input_data,
name="input_data",
dtype=tf.int32)
target_data = tf.convert_to_tensor(target_data,
name="target_data",
dtype=tf.int32)
weight_data = tf.convert_to_tensor(weight_data,
name="weight_data",
dtype=tf.float32)
batch_len = tf.size(input_data)
#print('batch_len=',end='')
#batch_len = tf.Print(batch_len,[batch_len])
epoch_size = batch_len // batch_size // num_steps
#epoch_size = tf.Print(epoch_size,[epoch_size])
assertion = tf.assert_positive(
epoch_size, message="epoch_size == 0, decrease num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(input_data, [i * batch_size, 0],
[(i + 1) * batch_size, num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(target_data, [i * batch_size, 0],
[(i + 1) * batch_size, num_steps])
y.set_shape([batch_size, num_steps])
z = tf.strided_slice(weight_data, [i * batch_size, 0],
[(i + 1) * batch_size, num_steps])
z.set_shape([batch_size, num_steps])
return x, y, z
def __init__(self,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="Laplace"):
"""Construct Laplace distribution with parameters `loc` and `scale`.
The parameters `loc` and `scale` must be shaped in a way that supports
broadcasting (e.g., `loc / scale` is a valid operation).
Args:
loc: Floating point tensor which characterizes the location (center)
of the distribution.
scale: Positive floating point tensor which characterizes the spread of
the distribution.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
TypeError: if `loc` and `scale` are of different dtype.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[loc, scale]) as name:
with tf.control_dependencies(
[tf.assert_positive(scale)] if validate_args else []):
self._loc = tf.identity(loc, name="loc")
self._scale = tf.identity(scale, name="scale")
tf.assert_same_float_dtype([self._loc, self._scale])
super(Laplace,
self).__init__(dtype=self._loc.dtype,
reparameterization_type=reparameterization.
FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._loc, self._scale],
name=name)
def snli_producer(data,config,name=None):
batch_size=config.batch_size
xmaxlen=config.xmaxlen
ymaxlen=config.ymaxlen
num_classes=config.num_classes
with tf.name_scope(name, "SNLIProducer"):
data_len=data._data_len
epoch_size =data_len//batch_size
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
data_x = tf.convert_to_tensor(data._x)
data_y = tf.convert_to_tensor(data._y)
data_labels = tf.convert_to_tensor(data._labels)
data_x_len = tf.convert_to_tensor(data._x_len)
data_y_len = tf.convert_to_tensor(data._y_len)
data_x_mask=tf.convert_to_tensor(data._x_mask)
data_y_mask=tf.convert_to_tensor(data._y_mask)
x = tf.strided_slice(data_x, [i*batch_size,0], [(i+1)*batch_size,xmaxlen])
y = tf.strided_slice(data_y, [i*batch_size,0], [(i+1)*batch_size,ymaxlen])
x_mask=tf.strided_slice(data_x_mask, [i*batch_size,0], [(i+1)*batch_size,xmaxlen])
y_mask=tf.strided_slice(data_y_mask, [i*batch_size,0], [(i+1)*batch_size,ymaxlen])
label = tf.strided_slice(data_labels, [i*batch_size,0], [(i+1)*batch_size,num_classes])
x_len = tf.strided_slice(data_x_len, [i*batch_size], [(i+1)*batch_size])
y_len = tf.strided_slice(data_y_len, [i*batch_size], [(i+1)*batch_size])
x.set_shape([batch_size,xmaxlen])
y.set_shape([batch_size,ymaxlen])
x_mask.set_shape([batch_size,xmaxlen])
y_mask.set_shape([batch_size,ymaxlen])
label.set_shape([batch_size,num_classes])
x_len.set_shape([batch_size])
y_len.set_shape([batch_size])
x_mask=tf.cast(x_mask,tf.float32)
y_mask=tf.cast(y_mask,tf.float32)
return x,y,label,x_len,y_len,data_len,x_mask,y_mask
def ptb_producer(raw_data, batch_size, num_steps, name=None):
"""Iterate on the raw exch data.
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from ptb_raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "EXCH2Producer",
[raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
name="raw_data",
dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def _assertions(self, x):
if not self.validate_args:
return []
x_shape = tf.shape(x)
is_matrix = tf.assert_rank_at_least(
x, 2, message="Input must have rank at least 2.")
is_square = tf.assert_equal(x_shape[-2],
x_shape[-1],
message="Input must be a square matrix.")
diag_part_x = tf.matrix_diag_part(x)
is_lower_triangular = tf.assert_equal(
tf.matrix_band_part(x, 0, -1), # Preserves triu, zeros rest.
tf.matrix_diag(diag_part_x),
message="Input must be lower triangular.")
is_positive_diag = tf.assert_positive(
diag_part_x,
message="Input must have all positive diagonal entries.")
return [is_matrix, is_square, is_lower_triangular, is_positive_diag]
def __init__(self,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="Cauchy"):
"""Construct Cauchy distributions.
The parameters `loc` and `scale` must be shaped in a way that supports
broadcasting (e.g. `loc + scale` is a valid operation).
Args:
loc: Floating point tensor; the modes of the distribution(s).
scale: Floating point tensor; the locations of the distribution(s).
Must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value "`NaN`" to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Raises:
TypeError: if `loc` and `scale` have different `dtype`.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[loc, scale]) as name:
with tf.control_dependencies([tf.assert_positive(scale)]
if validate_args else []):
dtype = dtype_util.common_dtype([loc, scale], tf.float32)
self._loc = tf.convert_to_tensor(loc, name="loc", dtype=dtype)
self._scale = tf.convert_to_tensor(scale, name="scale", dtype=dtype)
tf.assert_same_float_dtype([self._loc, self._scale])
super(Cauchy, self).__init__(
dtype=self._scale.dtype,
reparameterization_type=reparameterization.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._loc, self._scale],
name=name)
def ptb_producer(raw_data, batch_size, num_steps, name=None):
"""Iterate on the raw PTB data.
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from ptb_raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0 : batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]): # 依赖于 assertion为真 时才执行
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# 将数据进行切片，起始点是[0, i * num_steps]
# 终止点是[batch_size, (i + 1) * num_steps]
# 其中终止点的batch_size代表的是维度
# (i + 1) * num_steps代表的是数据的长度
# 这里即将data数据从第i * num_steps列开始，向后取(i + 1) * num_steps列，即一个num_steps的长度
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
# y的切法和x类似，只是y要向后一列移动一个单位，因为这里是根据上一个单词预测下一个单词
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def _build(self, inputs, mask_length, maxlen=None,):
"""
Apply a memory mask such that the values we mask result in being the
minimum possible value we can represent with a float32.
Taken from Sonnet Attention Module
:param inputs: [batch size, length], dtype=tf.float32
:param memory_mask: [batch_size] shape Tensor of ints indicating the
length of inputs
:param maxlen: Sets the maximum length of the sequence; if None infered
from inputs
:returns: [batch size, length] dim Tensor with the mask applied
"""
if len(mask_length.shape) != 1:
raise ValueError('Mask Length must be a 1-d Tensor, got %s' % mask_length.shape)
# [batch_size, length]
memory_mask = tf.sequence_mask(mask_length, maxlen=maxlen, name='SequenceMask')
inputs.shape.assert_is_compatible_with(memory_mask.shape)
num_remaining_memory_slots = tf.reduce_sum(
tf.cast(memory_mask, dtype=tf.int32), axis=[1])
with tf.control_dependencies([tf.assert_positive(
num_remaining_memory_slots)]):
# Get the numerical limits of a float
finfo = np.finfo(np.float32)
# If True = 1 = Keep that memory slot
kept_indices = tf.cast(memory_mask, dtype=tf.float32)
# Inverse
ignored_indices = tf.cast(tf.logical_not(memory_mask), dtype=tf.float32)
# If we keep the indices its the max float value else its the
# minimum float value. Then we can take the minimum
lower_bound = finfo.max * kept_indices + finfo.min * ignored_indices
slice_length = tf.reduce_max(mask_length)
# Return the elementwise
return tf.minimum(inputs[:, :slice_length],
lower_bound[:, :slice_length])
def calculate_kl_dist(posterior_collection, prior_collection):
'''
Calculate the KL (Kullback-Leibler) divergence between
the two factorized Gaussian distributions
'''
# Get the current parameters of the prior and posterior distributions:
posterior_tensors = tf.get_collection(posterior_collection)
prior_tensors = tf.get_collection(prior_collection)
post_mu_list = subset_with_substring(posterior_tensors, '_mu:')
prior_mu_list = subset_with_substring(prior_tensors, '_mu:')
post_sigma_sqr_list = subset_with_substring(posterior_tensors,
'_sigma_sqr:')
prior_sigma_sqr_list = subset_with_substring(prior_tensors, '_sigma_sqr:')
post_log_sigma_list = subset_with_substring(posterior_tensors,
'_log_sigma:')
prior_log_sigma_list = subset_with_substring(prior_tensors, '_log_sigma:')
# Calculate KL distance:
kl_dist = 0
for ii, _ in enumerate(post_mu_list):
mu_post = post_mu_list[ii]
mu_prior = prior_mu_list[ii]
sigma_sqr_post = post_sigma_sqr_list[ii]
sigma_sqr_prior = prior_sigma_sqr_list[ii]
log_sigma_post = post_log_sigma_list[ii]
log_sigma_prior = prior_log_sigma_list[ii]
# Calculate the contribution of current weights to the KL term:
p = 1e-9 # add small positive number to avoid division by zero due to numerical errors
curr_kl_dist = tf.reduce_sum(
log_sigma_prior - log_sigma_post + tf.divide(
tf.square(mu_post - mu_prior) +
sigma_sqr_post, 2 * sigma_sqr_prior + p)) - 0.5
curr_kl_dist = tf.nn.relu(
curr_kl_dist) # To avoid negative KL TODO: Find better fix
# debug assertion: KL must be positive:
with tf.control_dependencies([tf.assert_positive(curr_kl_dist + p)]):
kl_dist += curr_kl_dist
return kl_dist
def ptb_producer(raw_data, batch_size, num_steps, name=None):
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32) # 一维, 长度等于单词数量
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0: batch_size * batch_len], [batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(epoch_size, message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps], [batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1], [batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def _xy_producer(raw_data, batch_size, num_steps, name):
with tf.name_scope(name, values=[raw_data, batch_size, num_steps]):
# raw_data（=IDのリスト）から1階の整数テンソルを作成。
raw_data = tf.convert_to_tensor(raw_data,
name="raw_data",
dtype=tf.int32)
# raw_dataのワード数
data_len = tf.size(raw_data)
# raw_dataをbatch_size x batch_lenの行列にreshape
# - batch_size: 1バッチのサイズ（Mediumモデルだと20ワード）
# - batch_len: バッチ数
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
# 1エポックの学習回数
epoch_size = (batch_len - 1) // num_steps
# batch_len <= num_stepsだとepoch_sizeがゼロとなるためassertionを入れる
# control_dependenciesでepoch_sizeをassertionに依存させ、先にassertionを
# 評価させる。
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
# 以下のコードに対応する計算グラフを構築してreturn
# for i in range(epoch_size):
# x = data[:, i*num_steps:(i+1)*num_steps]
# y = data[:, (i*num_steps+1):((i+1)*num_steps+1)]
# yield x, y # ここで(x, y)をRNNに学習させる
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def __init__(self,
loc,
scale,
validate_args=False,
allow_nan_stats=True,
name="HalfCauchy"):
"""Construct a half-Cauchy distribution with `loc` and `scale`.
Args:
loc: Floating-point `Tensor`; the location(s) of the distribution(s).
scale: Floating-point `Tensor`; the scale(s) of the distribution(s).
Must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs. Default value: `False` (i.e. do not validate args).
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
Default value: `True`.
name: Python `str` name prefixed to Ops created by this class.
Default value: 'HalfCauchy'.
Raises:
TypeError: if `loc` and `scale` have different `dtype`.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[loc, scale]) as name:
with tf.control_dependencies([tf.assert_positive(scale)]
if validate_args else []):
self._loc = tf.identity(loc, name="loc")
self._scale = tf.identity(scale, name="scale")
tf.assert_same_float_dtype([self._loc, self._scale])
super(HalfCauchy, self).__init__(
dtype=self._scale.dtype,
reparameterization_type=tf.distributions.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._loc, self._scale],
name=name)
def __init__(self,
p_dropout,
loc,
scale,
validate_args=False,
allow_nan_stats=False,
name="DropoutNormal",
*args,
**kwargs):
"""Initialise DropoutNormal random variable
Parameters
------
p_dropout: tf.Tensor
loc: tf.Tensor
scale: tf.Tensor
"""
parameters = {'p_dropout': p_dropout, 'loc': loc, 'scale': scale}
with tf.name_scope(name, values=[p_dropout, loc, scale]):
with tf.control_dependencies(
[tf.assert_positive(scale)] if validate_args else []):
self._p_dropout = tf.identity(p_dropout, name="p_dropout")
self._loc = tf.identity(loc, name="loc")
self._scale = tf.identity(scale, name="scale")
super(DropoutNormal, self).__init__(
dtype=self._loc.dtype,
validate_args=False,
allow_nan_stats=False,
# is_continuous = True,
# is_reparametrized = True,
reparameterization_type=tf.contrib.distributions.
FULLY_REPARAMETERIZED,
parameters=parameters,
graph_parents=[self._p_dropout, self._loc, self._scale],
name=name,
*args,
**kwargs)
self._kwargs = parameters
def __init__(self,
df,
validate_args=False,
allow_nan_stats=True,
name="Chi2"):
"""Construct Chi2 distributions with parameter `df`.
Args:
df: Floating point tensor, the degrees of freedom of the
distribution(s). `df` must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or
more of the statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
"""
parameters = dict(locals())
# Even though all stats of chi2 are defined for valid parameters, this is
# not true in the parent class "gamma." therefore, passing
# allow_nan_stats=True
# through to the parent class results in unnecessary asserts.
with tf.name_scope(name, values=[df]) as name:
df = tf.convert_to_tensor(df,
name="df",
dtype=dtype_util.common_dtype(
[df], preferred_dtype=tf.float32))
with tf.control_dependencies([
tf.assert_positive(df),
] if validate_args else []):
self._df = tf.identity(df, name="df")
super(Chi2, self).__init__(concentration=0.5 * self._df,
rate=0.5,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name)
self._parameters = parameters
def ptb_producer(raw_data, batch_size, num_steps, name=None):
"""Iterate on the raw PTB data.
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from ptb_raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second
element of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
dtype=tf.int32,
name="raw_data")
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
ds = (tf.data.Dataset.range(tf.cast(
epoch_size, tf.int64)).map(lambda epoch_index: parse_record_fn(
epoch_index, data, num_steps, batch_size)).repeat(1).prefetch(
1))
return ds
def ptb_producer(raw_data, batch_size, num_steps, name=None):
''' Iterate on the raw PTB data
This chunks up raw_data into batches of examples and returns Tensors that are
drawn from these batches.
Args:
raw_data: one of the raw_data outputs from ptb_raw_data
batch_size: int, the batch size, samples/round of train
num_steps: int, the number of unrolls, or the number of batch
name: the name of this operation
Returns:
A pair of tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high
'''
with tf.name_scope(name, 'PTBProducer', [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data, dtype=tf.int32)
data_len = tf.size(raw_data)
batch_num = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_num],
[batch_size, batch_num])
epoch_size = (batch_num -
1) // num_steps # How many batches in one epoch
assertion = tf.assert_positive(epoch_size, message='epoch_size==0')
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name='epoch_size')
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def __init__(self,
scale,
validate_args=False,
allow_nan_stats=True,
name="Horseshoe"):
"""Construct a Horseshoe distribution with `scale`.
Args:
scale: Floating point tensor; the scales of the distribution(s).
Must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs. Default value: `False` (i.e., do not validate args).
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value "`NaN`" to indicate the
result is undefined. When `False`, an exception is raised if one or more
of the statistic's batch members are undefined.
Default value: `True`.
name: Python `str` name prefixed to Ops created by this class.
Default value: 'Horseshoe'.
"""
parameters = dict(locals())
with tf.name_scope(name, values=[scale]) as name:
dtype = dtype_util.common_dtype([scale],
preferred_dtype=tf.float32)
scale = tf.convert_to_tensor(scale, name="scale", dtype=dtype)
with tf.control_dependencies(
[tf.assert_positive(scale)] if validate_args else []):
self._scale = tf.identity(scale, name="scale")
self._half_cauchy = HalfCauchy(loc=tf.zeros([], dtype=dtype),
scale=tf.ones([], dtype=dtype),
allow_nan_stats=True)
super(Horseshoe, self).__init__(
dtype=self._scale.dtype,
reparameterization_type=reparameterization.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
graph_parents=[self._scale],
name=name)
def body(upper, lower, eta, fx):
# Evaluate local_fn(eta) := fn(x + eta*proj)
eta = lower + (upper - lower) * tf.random_uniform([],
dtype=dtype)
fx = local_fn(eta)
# Shrink envelope (tighten upper xor lower bound)
upper, lower = tf.cond\
(
eta > 0.0,
lambda: (eta, lower),
lambda: (upper, eta),
)
# Test boundary case: eta==0.0
test_envelope = tf.assert_positive(
tf.abs(eta),
name='test_envelope',
message='Sampling envelope collapsed to zero.')
with tf.control_dependencies([test_envelope]):
return upper, lower, eta, fx
def ptb_producer(raw_data, batch_size, num_steps, name=None):
# raw PTB 데이터에 대해 반복한다.
# raw_data를 batches of examples로 변환하고 이 batches들로부터 얻은 Tensors를 반환한다.
# 인자들(Args):
# raw_data: ptb_raw_data로부터 얻은 raw data outputs 중 하나.
# batch_size: int, 배치 크기(the batch size).
# num_steps: int, 학습하는 스텝의 크기(the number of unrolls).
# name: operation의 이름 (optional).
# 반환값들(Returns):
# [batch_size, num_steps]로 표현된 Tensors 쌍(pair). tuple의 두번째 element는
# 한 step만큼 time-shifted된 같은 데이터이다.
# 에러값 발생(Raises):
# tf.errors.InvalidArgumentError: batch_size나 num_steps가 너무 크면 발생한다.
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
name="raw_data",
dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def batch_data_producer(raw_data, batch_size, num_steps, name=None):
"""Iterate on the raw data.
This chunks up lyric_raw_data into batches of examples and returns Tensors that
are drawn from these batches.
Args:
raw_data: one of the raw data outputs from raw_data.
batch_size: int, the batch size.
num_steps: int, the number of unrolls.
name: the name of this operation (optional).
Returns:
A pair of Tensors, each shaped [batch_size, num_steps]. The second element
of the tuple is the same data time-shifted to the right by one.
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "batch_data_producer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0 : batch_size * batch_len],
[batch_size, batch_len]) # this may throw away some data.
epoch_size = (batch_len - 1) // num_steps # number of batches per unroll step
assertion = tf.assert_positive(epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
# this is to define the push a variable into the name scope.
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# x and y are exactly offseted just by one, not by one batch!!!!
x = tf.slice(data, [0, i * num_steps], [batch_size, num_steps])
y = tf.slice(data, [0, i * num_steps + 1], [batch_size, num_steps])
return x, y
def ptb_producer(raw_data, batch_size, num_steps, name=None):
"""
返回数据和对应的标签
This chunks up raw_data into batches of examples and returns Tensors that
are drawn from these batches.
:param raw_data: one of the raw data from ptb_raw_data.[w1.w2,w3,,,,]
:param batch_size: 要将数据分为多少批
:param num_steps: the number of unrolls,就是num_steps个单词组成一句话
:param name:the name of this operation (optional).
:return:
Raises:
tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.
"""
with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
name='raw_data',
dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size # 每批有多少个
# 上面用的是整除‘//’，所以会多对几个word，然后raw_data[0:batch_size * batch_len]是去掉这几个
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps ")
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
# 切割,strided_slice(data, start, end), [start, end)，前闭后开
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
# 下面构造target，每个当前word的target（‘类别’或者'ground truth'）就是他的下一个word，语言模型！
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def ptb_producer(raw_data, sequence_length, batch_size, num_steps, name=None):
print("Producing batch...")
with tf.name_scope(name, "PTBProducer",
[raw_data, sequence_length, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
name="raw_data",
dtype=tf.int32)
sequence_length = tf.convert_to_tensor(sequence_length,
name="sequence_length",
dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
sequence_length = tf.reshape(
sequence_length[0:batch_size * (batch_len // num_steps)],
[batch_size, batch_len // num_steps])
epoch_size = (batch_len) // num_steps
assertion = tf.assert_positive(
epoch_size,
message="epoch_size == 0, decrease batch_size or num_steps")
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name="epoch_size")
i = tf.train.range_input_producer(epoch_size, shuffle=True).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
seq_length = tf.strided_slice(sequence_length, [0, i],
[batch_size, i + 1])
seq_length.set_shape([batch_size, 1])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
print(y)
print("Producing batch finish")
return x, y, seq_length
def __init__(self,
loc,
log_scale=None,
scale=None,
validate_args=False,
allow_nan_stats=True):
parameters = dict(locals())
assert log_scale.shape.ndims == 2
assert (log_scale is None) != (scale is None)
with tf.name_scope(self.__class__.__name__):
if log_scale is not None:
loc = tf.identity(loc, name="loc")
scale = tf.exp(log_scale, name="scale")
log_scale = tf.identity(log_scale, name="log_scale")
elif scale is not None:
with tf.control_dependencies([tf.assert_positive(scale)]):
loc = tf.identity(loc, name="loc")
scale = tf.identity(scale, name="scale")
log_scale = tf.log(scale, name="log_scale")
assert loc.dtype.base_dtype == tf.float32 or loc.dtype.base_dtype == tf.float64
assert loc.dtype.base_dtype == log_scale.dtype.base_dtype == scale.dtype.base_dtype
self.loc = loc # [batch_size, self.dim]
self.log_scale = log_scale # [batch_size, self.dim] or [1, self.dim]
self.scale = scale # [batch_size, self.dim] or [1, self.dim]
self.dim = self.loc.shape.as_list()[1]
super().__init__(
dtype=self.loc.dtype,
reparameterization_type=tf.distributions.FULLY_REPARAMETERIZED,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
)
def add_noise(data, bin_widths):
"""Adds zero-mean uniform noise to the data.
Parameters
----------
data : Tensor
4D tensor with data-type `tf.float32`.
Data.
bin_widths : Tensor
1D tensor with data-type `tf.float32`.
Quantization bin widths. `bin_widths[i]`
corresponds to the length of the support
of the zero-mean uniform noise that is
added to `tf.slice(data, [0, 0, 0, i], [-1, -1, -1, 1])`.
Returns
-------
Tensor
4D tensor with data-type `tf.float32`.
Data with additive zero-mean uniform noise.
Raises
------
InvalidArgumentError
If a quantization bin width is not
strictly positive.
"""
# The shape of `data` does not change
# while running the graph. Therefore,
# the static shape of `data` is used.
shape_data = data.get_shape().as_list()
with tf.control_dependencies([tf.assert_positive(bin_widths)]):
tiled_bin_widths = tf.tile(
tf.reshape(bin_widths, [1, 1, 1, shape_data[3]]),
[shape_data[0], shape_data[1], shape_data[2], 1])
return data + tiled_bin_widths * tf.random_uniform(
shape_data, minval=-0.5, maxval=0.5, dtype=tf.float32)
def logit(x):
"""Evaluate :math:`\log(x / (1 - x))` elementwise.
Parameters
----------
x : tf.Tensor
A n-D tensor.
Returns
-------
tf.Tensor
A tensor of same shape as input.
Raises
------
InvalidArgumentError
If the input is not between :math:`(0,1)` elementwise.
"""
x = tf.convert_to_tensor(x)
dependencies = [tf.assert_positive(x), tf.assert_less(x, 1.0)]
x = control_flow_ops.with_dependencies(dependencies, x)
return tf.log(x) - tf.log(1.0 - x)
def __init__(self,
df,
validate_args=False,
allow_nan_stats=True,
name="Chi"):
"""Construct Chi distributions with parameter `df`.
Args:
df: Floating point tensor, the degrees of freedom of the
distribution(s). `df` must contain only positive values.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value `NaN` to indicate the result
is undefined. When `False`, an exception is raised if one or more of the
statistic's batch members are undefined.
name: Python `str` name prefixed to Ops created by this class.
Default value: `'Chi'`.
"""
with tf.name_scope(name, values=[df]) as name:
df = tf.convert_to_tensor(df,
name="df",
dtype=dtype_util.common_dtype(
[df], preferred_dtype=tf.float32))
validation_assertions = [tf.assert_positive(df)
] if validate_args else []
with tf.control_dependencies(validation_assertions):
self._df = tf.identity(df, name="df")
super(Chi, self).__init__(
distribution=chi2.Chi2(df=self._df,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name),
bijector=invert_bijector.Invert(square_bijector.Square()))
def build_cross_entropy_loss(logits, gold):
"""Constructs a cross entropy from logits and one-hot encoded gold labels.
Supports skipping rows where the gold label is the magic -1 value.
Args:
logits: float Tensor of scores.
gold: int Tensor of one-hot labels.
Returns:
cost, correct, total: the total cost, the total number of correctly
predicted labels, and the total number of valid labels.
"""
valid = tf.reshape(tf.where(tf.greater(gold, -1)), [-1])
gold = tf.gather(gold, valid)
logits = tf.gather(logits, valid)
correct = tf.reduce_sum(tf.to_int32(tf.nn.in_top_k(logits, gold, 1)))
total = tf.size(gold)
with tf.control_dependencies([tf.assert_positive(total)]):
cost = tf.reduce_sum(
tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=tf.cast(gold, tf.int64), logits=logits)) / tf.cast(
total, tf.float32)
return cost, correct, total
def batch_generator(self, name):
"""Generates a batch of samples.
Args:
name: name scope.
Returns:
A tensor pair: x [batch_size, 1], y [batch_size, 1].
"""
with tf.name_scope(name):
inputs = tf.convert_to_tensor(self._inputs,
name='inputs',
dtype=tf.float32)
targets = tf.convert_to_tensor(self._targets,
name='targets',
dtype=tf.float32)
data_len = tf.size(inputs)
num_batches = data_len // self.batch_size
inputs = tf.reshape(inputs[0:self.batch_size * num_batches],
[self.batch_size, num_batches])
targets = tf.reshape(targets[0:self.batch_size * num_batches],
[self.batch_size, num_batches])
assertion = tf.assert_positive(num_batches,
message='num_batches==0!')
with tf.control_dependencies([assertion]):
num_batches = tf.identity(num_batches,
name='num_batches_in_an_epoch')
i = tf.train.range_input_producer(limit=num_batches,
shuffle=False).dequeue()
x = tf.strided_slice(inputs, [0, i], [self.batch_size, i + 1])
x.set_shape([self.batch_size, 1])
y = tf.strided_slice(targets, [0, i], [self.batch_size, i + 1])
y.set_shape([self.batch_size, 1])
return x, y
def generate_data(raw_data, batch_size, num_steps, name=None):
with tf.name_scope(name, 'DataGenerator',
[raw_data, batch_size, num_steps]):
raw_data = tf.convert_to_tensor(raw_data,
name='raw_data',
dtype=tf.int32)
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(raw_data[0:batch_size * batch_len],
[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
assertion = tf.assert_positive(epoch_size, message='epoch_size == 0')
with tf.control_dependencies([assertion]):
epoch_size = tf.identity(epoch_size, name='epoch_size')
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
x = tf.strided_slice(data, [0, i * num_steps],
[batch_size, (i + 1) * num_steps])
x.set_shape([batch_size, num_steps])
y = tf.strided_slice(data, [0, i * num_steps + 1],
[batch_size, (i + 1) * num_steps + 1])
y.set_shape([batch_size, num_steps])
return x, y
def __init__(self,
skewness=None,
tailweight=None,
validate_args=False,
name="SinhArcsinh"):
"""Instantiates the `SinhArcsinh` bijector.
Args:
skewness: Skewness parameter. Float-type `Tensor`. Default is `0`
of type `float32`.
tailweight: Tailweight parameter. Positive `Tensor` of same `dtype` as
`skewness` and broadcastable `shape`. Default is `1` of type `float32`.
validate_args: Python `bool` indicating whether arguments should be
checked for correctness.
name: Python `str` name given to ops managed by this object.
"""
self._graph_parents = []
self._name = name
self._validate_args = validate_args
with self._name_scope("init", values=[skewness, tailweight]):
tailweight = 1. if tailweight is None else tailweight
skewness = 0. if skewness is None else skewness
self._skewness = tf.convert_to_tensor(skewness, name="skewness")
self._tailweight = tf.convert_to_tensor(tailweight,
name="tailweight",
dtype=self._skewness.dtype)
tf.assert_same_float_dtype([self._skewness, self._tailweight])
if validate_args:
self._tailweight = control_flow_ops.with_dependencies([
tf.assert_positive(
self._tailweight,
message="Argument tailweight was not positive")
], self._tailweight)
super(SinhArcsinh, self).__init__(forward_min_event_ndims=0,
validate_args=validate_args,
name=name)