def bottleneck(self, x, bottleneck_bits=None): # pylint: disable=arguments-differ
if bottleneck_bits is not None:
old_bottleneck_bits = self.hparams.bottleneck_bits
self.hparams.bottleneck_bits = bottleneck_bits
res, loss = discretization.parametrized_bottleneck(x, self.hparams)
if bottleneck_bits is not None:
self.hparams.bottleneck_bits = old_bottleneck_bits
return res, loss
def bottleneck(self, x, bottleneck_size=None):
if bottleneck_size is not None:
old_bottleneck_size = self.hparams.bottleneck_size
self.hparams.bottleneck_size = bottleneck_size
res = discretization.parametrized_bottleneck(x, self.hparams)
if bottleneck_size is not None:
self.hparams.bottleneck_size = old_bottleneck_size
return res
def bottleneck_layer(targets_c, hparams):
"""Compute latents from compressed targets."""
latents_discrete_hot, extra_loss = discretization.parametrized_bottleneck(
targets_c, hparams)
latents_dense = discretization.parametrized_unbottleneck(
latents_discrete_hot, hparams.hidden_size, hparams)
latents_discrete = tf.argmax(latents_discrete_hot, axis=-1)
if DO_SUMMARIES:
tf.summary.histogram("b0", tf.reshape(latents_discrete, [-1]))
return latents_dense, latents_discrete, extra_loss
def bottleneck_layer(targets_c, hparams):
"""Compute latents from compressed targets."""
latents_discrete_hot, extra_loss = discretization.parametrized_bottleneck(
targets_c, hparams)
latents_dense = discretization.parametrized_unbottleneck(
latents_discrete_hot, hparams.hidden_size, hparams)
latents_dense = targets_c + tf.stop_gradient(latents_dense - targets_c)
latents_discrete = tf.argmax(latents_discrete_hot, axis=-1)
if DO_SUMMARIES:
tf.summary.histogram("b0", tf.reshape(latents_discrete, [-1]))
return latents_dense, latents_discrete_hot, extra_loss
def bottleneck(self, x):
hparams = self.hparams
noise = hparams.bottleneck_noise
hparams.bottleneck_noise = 0.0 # We'll add noise below.
x = discretization.parametrized_bottleneck(x, hparams)
hparams.bottleneck_noise = noise
if hparams.mode == tf.estimator.ModeKeys.TRAIN:
# We want a number p such that p^bottleneck_size = 1 - noise.
# So log(p) * bottleneck_size = log(noise)
log_p = tf.log(1 - float(noise) / 2) / float(hparams.bottleneck_size)
# Probabilities of flipping are p, p^2, p^3, ..., p^bottleneck_size.
noise_mask = 1.0 - tf.exp(tf.cumsum(tf.zeros_like(x) + log_p, axis=-1))
# Having the no-noise mask, we can make noise just uniformly at random.
ordered_noise = tf.random_uniform(tf.shape(x))
# We want our noise to be 1s at the start and random {-1, 1} bits later.
ordered_noise = tf.to_float(tf.less(noise_mask, ordered_noise))
# Now we flip the bits of x on the noisy positions (ordered and normal).
x *= 2.0 * ordered_noise - 1
return x
def bottleneck(self, x): # pylint: disable=arguments-differ
hparams = self.hparams
if hparams.unordered:
return super(AutoencoderOrderedDiscrete, self).bottleneck(x)
noise = hparams.bottleneck_noise
hparams.bottleneck_noise = 0.0 # We'll add noise below.
x, loss = discretization.parametrized_bottleneck(x, hparams)
hparams.bottleneck_noise = noise
if hparams.mode == tf.estimator.ModeKeys.TRAIN:
# We want a number p such that p^bottleneck_bits = 1 - noise.
# So log(p) * bottleneck_bits = log(noise)
log_p = tf.log(1 - float(noise) / 2) / float(hparams.bottleneck_bits)
# Probabilities of flipping are p, p^2, p^3, ..., p^bottleneck_bits.
noise_mask = 1.0 - tf.exp(tf.cumsum(tf.zeros_like(x) + log_p, axis=-1))
# Having the no-noise mask, we can make noise just uniformly at random.
ordered_noise = tf.random_uniform(tf.shape(x))
# We want our noise to be 1s at the start and random {-1, 1} bits later.
ordered_noise = tf.to_float(tf.less(noise_mask, ordered_noise))
# Now we flip the bits of x on the noisy positions (ordered and normal).
x *= 2.0 * ordered_noise - 1
return x, loss
def bottleneck(self, x):
hparams = self._hparams
x = discretization.parametrized_bottleneck(x, hparams)
if hparams.mode == tf.estimator.ModeKeys.TRAIN:
# In the ordered case, we'll have no noise on top bits, let's make a mask.
# Start with randomly uniformly choosing numbers [0, number_of_bits) where
# the number of bits in our case is bottleneck size. We pick separately
# for every position and batch just to keep it varied.
no_noise_mask = tf.random_uniform(common_layers.shape_list(x)[:-1])
no_noise_mask *= hparams.bottleneck_size
# Now let's make a 1-hot vector that is 1 on the index i from which on
# we want to be noisy and 0 everywhere else.
no_noise_mask = tf.one_hot(tf.to_int32(no_noise_mask),
hparams.bottleneck_size)
# Use tf.cumsum to make the mask (0 before index i, 1 after index i).
no_noise_mask = tf.cumsum(no_noise_mask, axis=-1)
# Having the no-noise mask, we can make noise just uniformly at random.
ordered_noise = tf.random_uniform(tf.shape(x)) * no_noise_mask
# We want our noise to be 1s at the start and random {-1, 1} bits later.
ordered_noise = 2.0 * tf.to_float(tf.less(ordered_noise,
0.5)) - 1.0
# Now we flip the bits of x on the noisy positions (ordered and normal).
x *= ordered_noise
return x
