def recall_at_k(labels, predictions, k): ''' Compute recall at position k. :param labels: shape=(num_examples,), dtype=tf.int64 :param predictions: logits of shape=(num_examples, num_classes) :param k: recall position :return: recall at position k Example: labels = tf.constant([0, 1, 1], dtype=tf.int64) predictions = tf.constant([[0.1, 0.2, 0.3], [3, 5, 2], [0.3, 0.4, 0.7]]) recall_at_k(labels, predictions, 2) # recall_at_k(labels, predictions, 2) = 0.6667 ''' labels = expand_dims(labels, axis=1) _, predictions_idx = nn.top_k(predictions, k) predictions_idx = math_ops.to_int64(predictions_idx) tp = sets.set_size(sets.set_intersection(predictions_idx, labels)) tp = math_ops.to_double(tp) tp = math_ops.reduce_sum(tp) fn = sets.set_size( sets.set_difference(predictions_idx, labels, aminusb=False)) fn = math_ops.to_double(fn) fn = math_ops.reduce_sum(fn) recall = math_ops.div(tp, math_ops.add(tp, fn), name='recall_at_k') return recall
def mean_only_frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model from activations. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. In this variant, we only compute the difference between the means of the fitted Gaussians. The computation leads to O(n) vs. O(n^2) memory usage, yet still retains much of the same information as FID. Args: real_activations: 2D array of activations of real images of size [num_images, num_dims] to use to compute Frechet Inception distance. generated_activations: 2D array of activations of generated images of size [num_images, num_dims] to use to compute Frechet Inception distance. Returns: The mean-only Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute means of activations. m = math_ops.reduce_mean(real_activations, 0) m_w = math_ops.reduce_mean(generated_activations, 0) # Next the distance between means. mean = math_ops.reduce_sum(math_ops.squared_difference( m, m_w)) # Equivalent to L2 but more stable. mofid = mean if activations_dtype != dtypes.float64: mofid = math_ops.cast(mofid, activations_dtype) return mofid
def mean_only_frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model from activations. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. In this variant, we only compute the difference between the means of the fitted Gaussians. The computation leads to O(n) vs. O(n^2) memory usage, yet still retains much of the same information as FID. Args: real_activations: 2D array of activations of real images of size [num_images, num_dims] to use to compute Frechet Inception distance. generated_activations: 2D array of activations of generated images of size [num_images, num_dims] to use to compute Frechet Inception distance. Returns: The mean-only Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute means of activations. m = math_ops.reduce_mean(real_activations, 0) m_w = math_ops.reduce_mean(generated_activations, 0) # Next the distance between means. mean = math_ops.reduce_sum( math_ops.squared_difference(m, m_w)) # Equivalent to L2 but more stable. mofid = mean if activations_dtype != dtypes.float64: mofid = math_ops.cast(mofid, activations_dtype) return mofid
def approximate_duality_gap(self): """Add operations to compute the approximate duality gap. Returns: An Operation that computes the approximate duality gap over all examples. """ (primal_loss, dual_loss, example_weights) = _sdca_ops.sdca_training_stats( container=self._container, solver_uuid=self._solver_uuid) # Note that example_weights is guaranteed to be positive by # sdca_training_stats so dividing by it is safe. return (primal_loss + dual_loss + math_ops.to_double(self._l1_loss()) + (2.0 * math_ops.to_double(self._l2_loss( self._symmetric_l2_regularization())))) / example_weights
def approximate_duality_gap(self): """Add operations to compute the approximate duality gap. Returns: An Operation that computes the approximate duality gap over all examples. """ with name_scope('sdca/approximate_duality_gap'): _, values_list = self._hashtable.export_sharded() shard_sums = [] for values in values_list: with ops.device(values.device): # For large tables to_double() below allocates a large temporary # tensor that is freed once the sum operation completes. To reduce # peak memory usage in cases where we have multiple large tables on a # single device, we serialize these operations. # Note that we need double precision to get accurate results. with ops.control_dependencies(shard_sums): shard_sums.append( math_ops.reduce_sum(math_ops.to_double(values), 0)) summed_values = math_ops.add_n(shard_sums) primal_loss = summed_values[1] dual_loss = summed_values[2] example_weights = summed_values[3] # Note: we return NaN if there are no weights or all weights are 0, e.g. # if no examples have been processed return (primal_loss + dual_loss + self._l1_loss() + (2.0 * self._l2_loss(self._symmetric_l2_regularization())) ) / example_weights
def approximate_duality_gap(self): """Add operations to compute the approximate duality gap. Returns: An Operation that computes the approximate duality gap over all examples. """ summed_values = self._hashtable.values_reduce_sum() primal_loss = summed_values[1] dual_loss = summed_values[2] example_weights = summed_values[3] # TODO(andreasst): what about handle examples_weights == 0? return ( primal_loss + dual_loss + math_ops.to_double(self._l1_loss()) + (2.0 * math_ops.to_double(self._l2_loss(self._symmetric_l2_regularization()))) ) / example_weights
def frechet_classifier_distance_from_activations_new(real_activations, generated_activations): real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m = math_ops.reduce_mean(real_activations, 0) m_v = math_ops.reduce_mean(generated_activations, 0) num_examples = math_ops.to_double(array_ops.shape(real_activations)[0]) # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T real_centered = real_activations - m sigma = math_ops.matmul(real_centered, real_centered, transpose_a=True) / (num_examples - 1) gen_centered = generated_activations - m_v sigma_v = math_ops.matmul(gen_centered, gen_centered, transpose_a=True) / (num_examples - 1) # Find the Tr(sqrt(sigma sigma_v)) component of FID sqrt_trace_component = math_ops.trace( math_ops.sqrt(sigma) * sigma_v * math_ops.sqrt(sigma)) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component # Next the distance between means. mean = math_ops.square(linalg_ops.norm(m - m_v)) # This uses the L2 norm. fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def classifier_score(images, classifier_fn, num_batches=1): """Classifier score for evaluating a conditional generative model. This is based on the Inception Score, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1606.03498. In summary, this function calculates exp( E[ KL(p(y|x) || p(y)) ] ) which captures how different the network's classification prediction is from the prior distribution over classes. Args: images: Images to calculate the classifier score for. classifier_fn: A function that takes images and produces logits based on a classifier. num_batches: Number of batches to split `generated_images` in to in order to efficiently run them through the classifier network. Returns: The classifier score. A floating-point scalar of the same type as the output of `classifier_fn`. """ generated_images_list = array_ops.split(images, num_or_size_splits=num_batches) # Compute the classifier splits using the memory-efficient `map_fn`. logits = functional_ops.map_fn( fn=classifier_fn, elems=array_ops.stack(generated_images_list), parallel_iterations=1, back_prop=False, swap_memory=True, name='RunClassifier') logits = array_ops.concat(array_ops.unstack(logits), 0) logits.shape.assert_has_rank(2) # Use maximum precision for best results. logits_dtype = logits.dtype if logits_dtype != dtypes.float64: logits = math_ops.to_double(logits) p = nn_ops.softmax(logits) q = math_ops.reduce_mean(p, axis=0) kl = _kl_divergence(p, logits, q) kl.shape.assert_has_rank(1) log_score = math_ops.reduce_mean(kl) final_score = math_ops.exp(log_score) if logits_dtype != dtypes.float64: final_score = math_ops.cast(final_score, logits_dtype) return final_score
def frechet_classifier_distance_from_activations(real_activations, generated_activations): real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) m = math_ops.reduce_mean(real_activations, 0) m_w = math_ops.reduce_mean(generated_activations, 0) num_examples_real = math_ops.to_double( array_ops.shape(real_activations)[0]) num_examples_generated = math_ops.to_double( array_ops.shape(generated_activations)[0]) real_centered = real_activations - m sigma = math_ops.matmul(real_centered, real_centered, transpose_a=True) / (num_examples_real - 1) gen_centered = generated_activations - m_w sigma_w = math_ops.matmul(gen_centered, gen_centered, transpose_a=True) / (num_examples_generated - 1) sqrt_trace_component = trace_sqrt_product(sigma, sigma_w) trace = math_ops.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component mean = math_ops.reduce_sum(math_ops.squared_difference(m, m_w)) fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def classifier_score_from_logits(logits): """Classifier score for evaluating a generative model from logits. This method computes the classifier score for a set of logits. This can be used independently of the classifier_score() method, especially in the case of using large batches during evaluation where we would like precompute all of the logits before computing the classifier score. This technique is described in detail in https://arxiv.org/abs/1606.03498. In summary, this function calculates: exp( E[ KL(p(y|x) || p(y)) ] ) which captures how different the network's classification prediction is from the prior distribution over classes. Args: logits: Precomputed 2D tensor of logits that will be used to compute the classifier score. Returns: The classifier score. A floating-point scalar of the same type as the output of `logits`. """ logits.shape.assert_has_rank(2) # Use maximum precision for best results. logits_dtype = logits.dtype if logits_dtype != dtypes.float64: logits = math_ops.to_double(logits) p = nn_ops.softmax(logits) q = math_ops.reduce_mean(p, axis=0) kl = _kl_divergence(p, logits, q) kl.shape.assert_has_rank(1) log_score = math_ops.reduce_mean(kl) final_score = math_ops.exp(log_score) if logits_dtype != dtypes.float64: final_score = math_ops.cast(final_score, logits_dtype) return final_score
def classifier_score_from_logits(logits): logits.shape.assert_has_rank(2) # Use maximum precision for best results. logits_dtype = logits.dtype if logits_dtype != dtypes.float64: logits = math_ops.to_double(logits) p = nn_ops.softmax(logits) q = math_ops.reduce_mean(p, axis=0) kl = _kl_divergence(p, logits, q) kl.shape.assert_has_rank(1) log_score = math_ops.reduce_mean(kl) final_score = math_ops.exp(log_score) if logits_dtype != dtypes.float64: final_score = math_ops.cast(final_score, logits_dtype) return final_score
def classifier_score_from_logits(logits): """Classifier score for evaluating a conditional generative model. This is based on the Inception Score, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1606.03498. In summary, this function calculates exp( E[ KL(p(y|x) || p(y)) ] ) which captures how different the network's classification prediction is from the prior distribution over classes. Args: logits: A 2D Tensor of logits. Returns: The classifier score. A floating-point scalar of the same type as the output of `logits`. """ logits.shape.assert_has_rank(2) # Use maximum precision for best results. logits_dtype = logits.dtype if logits_dtype != dtypes.float64: logits = math_ops.to_double(logits) p = nn_ops.softmax(logits) q = math_ops.reduce_mean(p, axis=0) kl = _kl_divergence(p, logits, q) kl.shape.assert_has_rank(1) log_score = math_ops.reduce_mean(kl) final_score = math_ops.exp(log_score) if logits_dtype != dtypes.float64: final_score = math_ops.cast(final_score, logits_dtype) return final_score
def frechet_classifier_distance_from_activations(real_activations, generated_activations): """Classifier distance for evaluating a generative model. This methods computes the Frechet classifier distance from activations of real images and generated images. This can be used independently of the frechet_classifier_distance() method, especially in the case of using large batches during evaluation where we would like precompute all of the activations before computing the classifier distance. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calculates |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. Args: real_activations: 2D Tensor containing activations of real data. Shape is [batch_size, activation_size]. generated_activations: 2D Tensor containing activations of generated data. Shape is [batch_size, activation_size]. Returns: The Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m = math_ops.reduce_mean(real_activations, 0) m_w = math_ops.reduce_mean(generated_activations, 0) num_examples_real = math_ops.to_double( array_ops.shape(real_activations)[0]) num_examples_generated = math_ops.to_double( array_ops.shape(generated_activations)[0]) # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T real_centered = real_activations - m sigma = math_ops.matmul(real_centered, real_centered, transpose_a=True) / (num_examples_real - 1) gen_centered = generated_activations - m_w sigma_w = math_ops.matmul(gen_centered, gen_centered, transpose_a=True) / (num_examples_generated - 1) # Find the Tr(sqrt(sigma sigma_w)) component of FID sqrt_trace_component = trace_sqrt_product(sigma, sigma_w) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component # Next the distance between means. mean = math_ops.reduce_sum(math_ops.squared_difference( m, m_w)) # Equivalent to L2 but more stable. fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model. This is based on the Frechet Inception distance, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. Args: real_activations: Real images to use to compute Frechet Inception distance. generated_activations: Generated images to use to compute Frechet Inception distance. Returns: The Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m = math_ops.reduce_mean(real_activations, 0) m_v = math_ops.reduce_mean(generated_activations, 0) num_examples = math_ops.to_double(array_ops.shape(real_activations)[0]) # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T real_centered = real_activations - m sigma = math_ops.matmul( real_centered, real_centered, transpose_a=True) / (num_examples - 1) gen_centered = generated_activations - m_v sigma_v = math_ops.matmul( gen_centered, gen_centered, transpose_a=True) / (num_examples - 1) # Find the Tr(sqrt(sigma sigma_v)) component of FID sqrt_trace_component = trace_sqrt_product(sigma, sigma_v) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component # Next the distance between means. mean = math_ops.square(linalg_ops.norm(m - m_v)) # This uses the L2 norm. fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def frechet_classifier_distance(real_images, generated_images, classifier_fn, num_batches=1): """Classifier distance for evaluating a generative model. This is based on the Frechet Inception distance, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. Args: real_images: Real images to use to compute Frechet Inception distance. generated_images: Generated images to use to compute Frechet Inception distance. classifier_fn: A function that takes images and produces activations based on a classifier. num_batches: Number of batches to split images in to in order to efficiently run them through the classifier network. Returns: The Frechet Inception distance. A floating-point scalar of the same type as the output of `classifier_fn` """ real_images_list = array_ops.split(real_images, num_or_size_splits=num_batches) generated_images_list = array_ops.split(generated_images, num_or_size_splits=num_batches) imgs = array_ops.stack(real_images_list + generated_images_list) # Compute the activations using the memory-efficient `map_fn`. activations = functional_ops.map_fn(fn=classifier_fn, elems=imgs, parallel_iterations=1, back_prop=False, swap_memory=True, name='RunClassifier') activations_dtype = activations.dtype # Split the activations by the real and generated images. real_a, gen_a = array_ops.split(activations, [num_batches, num_batches], 0) # Ensure the activations have the right shapes. real_a = array_ops.concat(array_ops.unstack(real_a), 0) gen_a = array_ops.concat(array_ops.unstack(gen_a), 0) if activations_dtype != dtypes.float64: real_a = math_ops.to_double(real_a) gen_a = math_ops.to_double(gen_a) real_a.shape.assert_has_rank(2) gen_a.shape.assert_has_rank(2) # Compute mean and covariance matrices of activations. m = math_ops.reduce_mean(real_a, 0) m_v = math_ops.reduce_mean(gen_a, 0) num_examples = math_ops.to_double(array_ops.shape(real_a)[0]) # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T sigma = math_ops.matmul(real_a - m, real_a - m, transpose_a=True) / (num_examples - 1) sigma_v = math_ops.matmul(gen_a - m_v, gen_a - m_v, transpose_a=True) / (num_examples - 1) # Find the Tr(sqrt(sigma sigma_v)) component of FID sqrt_trace_component = trace_sqrt_product(sigma, sigma_v) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component # Next the distance between means. mean = math_ops.square(linalg_ops.norm(m - m_v)) # This uses the L2 norm. fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def percentile(x, q, axis=None, interpolation=None, keep_dims=False, validate_args=False, name=None): """Compute the `q`-th percentile of `x`. Given a vector `x`, the `q`-th percentile of `x` is the value `q / 100` of the way from the minimum to the maximum in a sorted copy of `x`. The values and distances of the two nearest neighbors as well as the `interpolation` parameter will determine the percentile if the normalized ranking does not match the location of `q` exactly. This function is the same as the median if `q = 50`, the same as the minimum if `q = 0` and the same as the maximum if `q = 100`. ```python # Get 30th percentile with default ('nearest') interpolation. x = [1., 2., 3., 4.] percentile(x, q=30.) ==> 2.0 # Get 30th percentile with 'lower' interpolation x = [1., 2., 3., 4.] percentile(x, q=30., interpolation='lower') ==> 1.0 # Get 100th percentile (maximum). By default, this is computed over every dim x = [[1., 2.] [3., 4.]] percentile(x, q=100.) ==> 4.0 # Treat the leading dim as indexing samples, and find the 100th quantile (max) # over all such samples. x = [[1., 2.] [3., 4.]] percentile(x, q=100., axis=[0]) ==> [3., 4.] ``` Compare to `numpy.percentile`. Args: x: Floating point `N-D` `Tensor` with `N > 0`. If `axis` is not `None`, `x` must have statically known number of dimensions. q: Scalar `Tensor` in `[0, 100]`. The percentile. axis: Optional `0-D` or `1-D` integer `Tensor` with constant values. The axis that hold independent samples over which to return the desired percentile. If `None` (the default), treat every dimension as a sample dimension, returning a scalar. interpolation : {"lower", "higher", "nearest"}. Default: "nearest" This optional parameter specifies the interpolation method to use when the desired quantile lies between two data points `i < j`: * lower: `i`. * higher: `j`. * nearest: `i` or `j`, whichever is nearest. keep_dims: Python `bool`. If `True`, the last dimension is kept with size 1 If `False`, the last dimension is removed from the output shape. validate_args: Whether to add runtime checks of argument validity. If False, and arguments are incorrect, correct behavior is not guaranteed. name: A Python string name to give this `Op`. Default is "percentile" Returns: A `(N - len(axis))` dimensional `Tensor` of same dtype as `x`, or, if `axis` is `None`, a scalar. Raises: ValueError: If argument 'interpolation' is not an allowed type. """ name = name or "percentile" allowed_interpolations = {"lower", "higher", "nearest"} if interpolation is None: interpolation = "nearest" else: if interpolation not in allowed_interpolations: raise ValueError("Argument 'interpolation' must be in %s. Found %s" % (allowed_interpolations, interpolation)) with ops.name_scope(name, [x, q]): x = ops.convert_to_tensor(x, name="x") # Double is needed here and below, else we get the wrong index if the array # is huge along axis. q = math_ops.to_double(q, name="q") _get_static_ndims(q, expect_ndims=0) if validate_args: q = control_flow_ops.with_dependencies([ check_ops.assert_rank(q, 0), check_ops.assert_greater_equal(q, math_ops.to_double(0.)), check_ops.assert_less_equal(q, math_ops.to_double(100.)) ], q) if axis is None: y = array_ops.reshape(x, [-1]) else: axis = ops.convert_to_tensor(axis, name="axis") check_ops.assert_integer(axis) axis_ndims = _get_static_ndims( axis, expect_static=True, expect_ndims_no_more_than=1) axis_const = tensor_util.constant_value(axis) if axis_const is None: raise ValueError( "Expected argument 'axis' to be statically available. Found: %s" % axis) axis = axis_const if axis_ndims == 0: axis = [axis] axis = [int(a) for a in axis] x_ndims = _get_static_ndims( x, expect_static=True, expect_ndims_at_least=1) axis = _make_static_axis_non_negative(axis, x_ndims) y = _move_dims_to_flat_end(x, axis, x_ndims) frac_at_q_or_above = 1. - q / 100. d = math_ops.to_double(array_ops.shape(y)[-1]) if interpolation == "lower": index = math_ops.ceil((d - 1) * frac_at_q_or_above) elif interpolation == "higher": index = math_ops.floor((d - 1) * frac_at_q_or_above) elif interpolation == "nearest": index = math_ops.round((d - 1) * frac_at_q_or_above) # If d is gigantic, then we would have d == d - 1, even in double... So # let's use max/min to avoid out of bounds errors. d = array_ops.shape(y)[-1] # d - 1 will be distinct from d in int32. index = clip_ops.clip_by_value(math_ops.to_int32(index), 0, d - 1) # Sort everything, not just the top 'k' entries, which allows multiple calls # to sort only once (under the hood) and use CSE. sorted_y = _sort_tensor(y) # result.shape = B result = sorted_y[..., index] result.set_shape(y.get_shape()[:-1]) if keep_dims: if axis is None: # ones_vec = [1, 1,..., 1], total length = len(S) + len(B). ones_vec = array_ops.ones( shape=[_get_best_effort_ndims(x)], dtype=dtypes.int32) result *= array_ops.ones(ones_vec, dtype=x.dtype) else: result = _insert_back_keep_dims(result, axis) return result
def percentile(x, q, axis=None, interpolation=None, keep_dims=False, validate_args=False, name=None): """Compute the `q`-th percentile of `x`. Given a vector `x`, the `q`-th percentile of `x` is the value `q / 100` of the way from the minimum to the maximum in a sorted copy of `x`. The values and distances of the two nearest neighbors as well as the `interpolation` parameter will determine the percentile if the normalized ranking does not match the location of `q` exactly. This function is the same as the median if `q = 50`, the same as the minimum if `q = 0` and the same as the maximum if `q = 100`. ```python # Get 30th percentile with default ('nearest') interpolation. x = [1., 2., 3., 4.] percentile(x, q=30.) ==> 2.0 # Get 30th percentile with 'lower' interpolation x = [1., 2., 3., 4.] percentile(x, q=30., interpolation='lower') ==> 1.0 # Get 100th percentile (maximum). By default, this is computed over every dim x = [[1., 2.] [3., 4.]] percentile(x, q=100.) ==> 4.0 # Treat the leading dim as indexing samples, and find the 100th quantile (max) # over all such samples. x = [[1., 2.] [3., 4.]] percentile(x, q=100., axis=[0]) ==> [3., 4.] ``` Compare to `numpy.percentile`. Args: x: Floating point `N-D` `Tensor` with `N > 0`. If `axis` is not `None`, `x` must have statically known number of dimensions. q: Scalar `Tensor` in `[0, 100]`. The percentile. axis: Optional `0-D` or `1-D` integer `Tensor` with constant values. The axis that hold independent samples over which to return the desired percentile. If `None` (the default), treat every dimension as a sample dimension, returning a scalar. interpolation : {"lower", "higher", "nearest"}. Default: "nearest" This optional parameter specifies the interpolation method to use when the desired quantile lies between two data points `i < j`: * lower: `i`. * higher: `j`. * nearest: `i` or `j`, whichever is nearest. keep_dims: Python `bool`. If `True`, the last dimension is kept with size 1 If `False`, the last dimension is removed from the output shape. validate_args: Whether to add runtime checks of argument validity. If False, and arguments are incorrect, correct behavior is not guaranteed. name: A Python string name to give this `Op`. Default is "percentile" Returns: A `(N - len(axis))` dimensional `Tensor` of same dtype as `x`, or, if `axis` is `None`, a scalar. Raises: ValueError: If argument 'interpolation' is not an allowed type. """ name = name or "percentile" allowed_interpolations = {"lower", "higher", "nearest"} if interpolation is None: interpolation = "nearest" else: if interpolation not in allowed_interpolations: raise ValueError( "Argument 'interpolation' must be in %s. Found %s" % (allowed_interpolations, interpolation)) with ops.name_scope(name, [x, q]): x = ops.convert_to_tensor(x, name="x") # Double is needed here and below, else we get the wrong index if the array # is huge along axis. q = math_ops.to_double(q, name="q") _get_static_ndims(q, expect_ndims=0) if validate_args: q = control_flow_ops.with_dependencies([ check_ops.assert_rank(q, 0), check_ops.assert_greater_equal(q, math_ops.to_double(0.)), check_ops.assert_less_equal(q, math_ops.to_double(100.)) ], q) if axis is None: y = array_ops.reshape(x, [-1]) else: axis = ops.convert_to_tensor(axis, name="axis") check_ops.assert_integer(axis) axis_ndims = _get_static_ndims(axis, expect_static=True, expect_ndims_no_more_than=1) axis_const = tensor_util.constant_value(axis) if axis_const is None: raise ValueError( "Expected argument 'axis' to be statically available. Found: %s" % axis) axis = axis_const if axis_ndims == 0: axis = [axis] axis = [int(a) for a in axis] x_ndims = _get_static_ndims(x, expect_static=True, expect_ndims_at_least=1) axis = _make_static_axis_non_negative(axis, x_ndims) y = _move_dims_to_flat_end(x, axis, x_ndims) frac_at_q_or_above = 1. - q / 100. d = math_ops.to_double(array_ops.shape(y)[-1]) if interpolation == "lower": index = math_ops.ceil((d - 1) * frac_at_q_or_above) elif interpolation == "higher": index = math_ops.floor((d - 1) * frac_at_q_or_above) elif interpolation == "nearest": index = math_ops.round((d - 1) * frac_at_q_or_above) # If d is gigantic, then we would have d == d - 1, even in double... So # let's use max/min to avoid out of bounds errors. d = array_ops.shape(y)[-1] # d - 1 will be distinct from d in int32. index = clip_ops.clip_by_value(math_ops.to_int32(index), 0, d - 1) # Sort everything, not just the top 'k' entries, which allows multiple calls # to sort only once (under the hood) and use CSE. sorted_y = _sort_tensor(y) # result.shape = B result = sorted_y[..., index] result.set_shape(y.get_shape()[:-1]) if keep_dims: if axis is None: # ones_vec = [1, 1,..., 1], total length = len(S) + len(B). ones_vec = array_ops.ones(shape=[_get_best_effort_ndims(x)], dtype=dtypes.int32) result *= array_ops.ones(ones_vec, dtype=x.dtype) else: result = _insert_back_keep_dims(result, axis) return result
def diagonal_only_frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model. This is based on the Frechet Inception distance, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 + (sigma + sigma_w - 2(sigma x sigma_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. In this variant, we compute diagonal-only covariance matrices. As a result, instead of computing an expensive matrix square root, we can do something much simpler, and has O(n) vs O(n^2) space complexity. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. Args: real_activations: Real images to use to compute Frechet Inception distance. generated_activations: Generated images to use to compute Frechet Inception distance. Returns: The diagonal-only Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. Raises: ValueError: If the shape of the variance and mean vectors are not equal. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m, var = nn_impl.moments(real_activations, axes=[0]) m_w, var_w = nn_impl.moments(generated_activations, axes=[0]) actual_shape = var.get_shape() expected_shape = m.get_shape() if actual_shape != expected_shape: raise ValueError('shape: {} must match expected shape: {}'.format( actual_shape, expected_shape)) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.reduce_sum( (var + var_w) - 2.0 * math_ops.sqrt(math_ops.multiply(var, var_w))) # Next the distance between means. mean = math_ops.reduce_sum( math_ops.squared_difference(m, m_w)) # Equivalent to L2 but more stable. dofid = trace + mean if activations_dtype != dtypes.float64: dofid = math_ops.cast(dofid, activations_dtype) return dofid
def frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model from activations. This methods computes the Frechet classifier distance from activations of real images and generated images. This can be used independently of the frechet_classifier_distance() method, especially in the case of using large batches during evaluation where we would like precompute all of the activations before computing the classifier distance. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. Args: real_activations: 2D Tensor containing activations of real data. Shape is [batch_size, activation_size]. generated_activations: 2D Tensor containing activations of generated data. Shape is [batch_size, activation_size]. Returns: The Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m = math_ops.reduce_mean(real_activations, 0) m_v = math_ops.reduce_mean(generated_activations, 0) num_examples = math_ops.to_double(array_ops.shape(real_activations)[0]) # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T real_centered = real_activations - m sigma = math_ops.matmul( real_centered, real_centered, transpose_a=True) / (num_examples - 1) gen_centered = generated_activations - m_v sigma_v = math_ops.matmul( gen_centered, gen_centered, transpose_a=True) / (num_examples - 1) # Find the Tr(sqrt(sigma sigma_v)) component of FID sqrt_trace_component = trace_sqrt_product(sigma, sigma_v) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component # Next the distance between means. mean = math_ops.square(linalg_ops.norm(m - m_v)) # This uses the L2 norm. fid = trace + mean if activations_dtype != dtypes.float64: fid = math_ops.cast(fid, activations_dtype) return fid
def diagonal_only_frechet_classifier_distance_from_activations( real_activations, generated_activations): """Classifier distance for evaluating a generative model. This is based on the Frechet Inception distance, but for an arbitrary classifier. This technique is described in detail in https://arxiv.org/abs/1706.08500. Given two Gaussian distribution with means m and m_w and covariance matrices C and C_w, this function calcuates |m - m_w|^2 + (sigma + sigma_w - 2(sigma x sigma_w)^(1/2)) which captures how different the distributions of real images and generated images (or more accurately, their visual features) are. Note that unlike the Inception score, this is a true distance and utilizes information about real world images. In this variant, we compute diagonal-only covariance matrices. As a result, instead of computing an expensive matrix square root, we can do something much simpler, and has O(n) vs O(n^2) space complexity. Note that when computed using sample means and sample covariance matrices, Frechet distance is biased. It is more biased for small sample sizes. (e.g. even if the two distributions are the same, for a small sample size, the expected Frechet distance is large). It is important to use the same sample size to compute frechet classifier distance when comparing two generative models. Args: real_activations: Real images to use to compute Frechet Inception distance. generated_activations: Generated images to use to compute Frechet Inception distance. Returns: The diagonal-only Frechet Inception distance. A floating-point scalar of the same type as the output of the activations. Raises: ValueError: If the shape of the variance and mean vectors are not equal. """ real_activations.shape.assert_has_rank(2) generated_activations.shape.assert_has_rank(2) activations_dtype = real_activations.dtype if activations_dtype != dtypes.float64: real_activations = math_ops.to_double(real_activations) generated_activations = math_ops.to_double(generated_activations) # Compute mean and covariance matrices of activations. m, var = nn_impl.moments(real_activations, axes=[0]) m_w, var_w = nn_impl.moments(generated_activations, axes=[0]) actual_shape = var.get_shape() expected_shape = m.get_shape() if actual_shape != expected_shape: raise ValueError('shape: {} must match expected shape: {}'.format( actual_shape, expected_shape)) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = math_ops.reduce_sum((var + var_w) - 2.0 * math_ops.sqrt(math_ops.multiply(var, var_w))) # Next the distance between means. mean = math_ops.reduce_sum(math_ops.squared_difference( m, m_w)) # Equivalent to L2 but more stable. dofid = trace + mean if activations_dtype != dtypes.float64: dofid = math_ops.cast(dofid, activations_dtype) return dofid