def bottleneck(self, x, bottleneck_bits=None): # pylint: disable=arguments-differ if bottleneck_bits is not None: old_bottleneck_bits = self.hparams.bottleneck_bits self.hparams.bottleneck_bits = bottleneck_bits res, loss = discretization.parametrized_bottleneck(x, self.hparams) if bottleneck_bits is not None: self.hparams.bottleneck_bits = old_bottleneck_bits return res, loss
def bottleneck(self, x, bottleneck_size=None): if bottleneck_size is not None: old_bottleneck_size = self.hparams.bottleneck_size self.hparams.bottleneck_size = bottleneck_size res = discretization.parametrized_bottleneck(x, self.hparams) if bottleneck_size is not None: self.hparams.bottleneck_size = old_bottleneck_size return res
def bottleneck_layer(targets_c, hparams): """Compute latents from compressed targets.""" latents_discrete_hot, extra_loss = discretization.parametrized_bottleneck( targets_c, hparams) latents_dense = discretization.parametrized_unbottleneck( latents_discrete_hot, hparams.hidden_size, hparams) latents_discrete = tf.argmax(latents_discrete_hot, axis=-1) if DO_SUMMARIES: tf.summary.histogram("b0", tf.reshape(latents_discrete, [-1])) return latents_dense, latents_discrete, extra_loss
def bottleneck_layer(targets_c, hparams): """Compute latents from compressed targets.""" latents_discrete_hot, extra_loss = discretization.parametrized_bottleneck( targets_c, hparams) latents_dense = discretization.parametrized_unbottleneck( latents_discrete_hot, hparams.hidden_size, hparams) latents_dense = targets_c + tf.stop_gradient(latents_dense - targets_c) latents_discrete = tf.argmax(latents_discrete_hot, axis=-1) if DO_SUMMARIES: tf.summary.histogram("b0", tf.reshape(latents_discrete, [-1])) return latents_dense, latents_discrete_hot, extra_loss
def bottleneck(self, x): hparams = self.hparams noise = hparams.bottleneck_noise hparams.bottleneck_noise = 0.0 # We'll add noise below. x = discretization.parametrized_bottleneck(x, hparams) hparams.bottleneck_noise = noise if hparams.mode == tf.estimator.ModeKeys.TRAIN: # We want a number p such that p^bottleneck_size = 1 - noise. # So log(p) * bottleneck_size = log(noise) log_p = tf.log(1 - float(noise) / 2) / float(hparams.bottleneck_size) # Probabilities of flipping are p, p^2, p^3, ..., p^bottleneck_size. noise_mask = 1.0 - tf.exp(tf.cumsum(tf.zeros_like(x) + log_p, axis=-1)) # Having the no-noise mask, we can make noise just uniformly at random. ordered_noise = tf.random_uniform(tf.shape(x)) # We want our noise to be 1s at the start and random {-1, 1} bits later. ordered_noise = tf.to_float(tf.less(noise_mask, ordered_noise)) # Now we flip the bits of x on the noisy positions (ordered and normal). x *= 2.0 * ordered_noise - 1 return x
def bottleneck(self, x): # pylint: disable=arguments-differ hparams = self.hparams if hparams.unordered: return super(AutoencoderOrderedDiscrete, self).bottleneck(x) noise = hparams.bottleneck_noise hparams.bottleneck_noise = 0.0 # We'll add noise below. x, loss = discretization.parametrized_bottleneck(x, hparams) hparams.bottleneck_noise = noise if hparams.mode == tf.estimator.ModeKeys.TRAIN: # We want a number p such that p^bottleneck_bits = 1 - noise. # So log(p) * bottleneck_bits = log(noise) log_p = tf.log(1 - float(noise) / 2) / float(hparams.bottleneck_bits) # Probabilities of flipping are p, p^2, p^3, ..., p^bottleneck_bits. noise_mask = 1.0 - tf.exp(tf.cumsum(tf.zeros_like(x) + log_p, axis=-1)) # Having the no-noise mask, we can make noise just uniformly at random. ordered_noise = tf.random_uniform(tf.shape(x)) # We want our noise to be 1s at the start and random {-1, 1} bits later. ordered_noise = tf.to_float(tf.less(noise_mask, ordered_noise)) # Now we flip the bits of x on the noisy positions (ordered and normal). x *= 2.0 * ordered_noise - 1 return x, loss
def bottleneck(self, x): hparams = self._hparams x = discretization.parametrized_bottleneck(x, hparams) if hparams.mode == tf.estimator.ModeKeys.TRAIN: # In the ordered case, we'll have no noise on top bits, let's make a mask. # Start with randomly uniformly choosing numbers [0, number_of_bits) where # the number of bits in our case is bottleneck size. We pick separately # for every position and batch just to keep it varied. no_noise_mask = tf.random_uniform(common_layers.shape_list(x)[:-1]) no_noise_mask *= hparams.bottleneck_size # Now let's make a 1-hot vector that is 1 on the index i from which on # we want to be noisy and 0 everywhere else. no_noise_mask = tf.one_hot(tf.to_int32(no_noise_mask), hparams.bottleneck_size) # Use tf.cumsum to make the mask (0 before index i, 1 after index i). no_noise_mask = tf.cumsum(no_noise_mask, axis=-1) # Having the no-noise mask, we can make noise just uniformly at random. ordered_noise = tf.random_uniform(tf.shape(x)) * no_noise_mask # We want our noise to be 1s at the start and random {-1, 1} bits later. ordered_noise = 2.0 * tf.to_float(tf.less(ordered_noise, 0.5)) - 1.0 # Now we flip the bits of x on the noisy positions (ordered and normal). x *= ordered_noise return x