def test_covariance_accuracy(self): """Checks accuracy of streaming_covariance.""" if tf.executing_eagerly(): # tf.placeholder() is not compatible with eager execution. return np.random.seed(0) num_batches = 128 batch_size = 32 dim = 32 data = np.random.randn(num_batches, batch_size, dim) placeholder = tf.compat.v1.placeholder(dtype=tf.float64, shape=(batch_size, dim)) value, update_op = eval_utils.streaming_covariance(placeholder) expected_result = np.cov(np.reshape(data, [num_batches * batch_size, dim]), rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) for i in range(num_batches): sess.run(update_op, feed_dict={placeholder: data[i]}) result = sess.run(value) self.assertAllClose(expected_result, result, rtol=1e-15, atol=1e-15)
def test_covariance_float32(self): """Checks handling of float32 values in streaming_covariance.""" x = [[1., 2.], [2., 1.]] result, update_op = eval_utils.streaming_covariance( x=tf.constant(x, dtype=tf.float32)) expected_result = np.cov(x, rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) self.assertAllClose(expected_result, update_op) self.assertAllClose(expected_result, result)
def test_covariance_simple(self): """Sanity check for streaming_covariance.""" if tf.executing_eagerly(): # streaming_covariance is not supported when eager execution is enabled. return x = [[1., 2.], [2., 1.]] result, update_op = eval_utils.streaming_covariance( tf.constant(x, dtype=tf.float64)) expected_result = np.cov(x, rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) self.assertAllClose(expected_result, update_op) self.assertAllClose(expected_result, result)
def test_covariance_float32(self): from tensorflow_gan.python.eval import eval_utils """Checks handling of float32 values in streaming_covariance.""" if tf.executing_eagerly(): # streaming_covariance is not supported when eager execution is enabled. return x = [[1., 2.], [2., 1.]] result, update_op = eval_utils.streaming_covariance( x=tf.constant(x, dtype=tf.float32)) expected_result = np.cov(x, rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) self.assertAllClose(expected_result, update_op) self.assertAllClose(expected_result, result)
def test_covariance_float32_with_y(self): """Checks handling of float32 values in streaming_covariance.""" x = [[1., 2.], [2., 1.]] y = [[1., 2.], [2., 1.]] result, update_op = eval_utils.streaming_covariance( x=tf.constant(x, dtype=tf.float32), y=tf.constant(y, dtype=tf.float32)) # We mulitiply by N/(N-1)=2 to get the unbiased estimator. # Note that we cannot use np.cov since it uses a different semantics for y. expected_result = 2. * tfp.stats.covariance(x, y) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) self.assertAllClose(expected_result, update_op) self.assertAllClose(expected_result, result)
def test_covariance_with_y(self): """Checks output of streaming_covariance given value for y.""" if tf.executing_eagerly(): # streaming_covariance is not supported when eager execution is enabled. return x = [[1., 2.], [2., 1.]] y = [[3., 3.], [1., 0.]] result, update_op = eval_utils.streaming_covariance( x=tf.constant(x, dtype=tf.float64), y=tf.constant(y, dtype=tf.float64)) # We mulitiply by N/(N-1)=2 to get the unbiased estimator. # Note that we cannot use np.cov since it uses a different semantics for y. expected_result = 2. * tfp.stats.covariance(x, y) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) self.assertAllClose(expected_result, update_op) self.assertAllClose(expected_result, result)
def test_covariance_batches(self): """Checks value consistency of streaming_covariance.""" np.random.seed(0) num_batches = 8 data = np.random.randn(num_batches, 4, 5) placeholder = tf.compat.v1.placeholder(dtype=tf.float64, shape=(4, 5)) value, update_op = eval_utils.streaming_covariance(placeholder) expected_result = np.cov(np.reshape(data, [num_batches * 4, 5]), rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) for i in range(num_batches): update_op_result = sess.run(update_op, feed_dict={placeholder: data[i]}) result = sess.run(value) self.assertAllClose(update_op_result, result) self.assertAllClose(expected_result, result)
def test_covariance_accuracy_with_y(self): """Checks accuracy of streaming_covariance with two input tensors.""" if tf.executing_eagerly(): # tf.placeholder() is not compatible with eager execution. return np.random.seed(0) num_batches = 128 batch_size = 32 dim = 32 x = np.random.randn(num_batches, batch_size, dim) y = np.random.randn(num_batches, batch_size, dim) placeholder_x = tf.compat.v1.placeholder(dtype=tf.float64, shape=(batch_size, dim)) placeholder_y = tf.compat.v1.placeholder(dtype=tf.float64, shape=(batch_size, dim)) value, update_op = eval_utils.streaming_covariance( placeholder_x, placeholder_y) # We mulitiply by N/(N-1) to get the unbiased estimator. # Note that we cannot use np.cov since it uses different semantics for y. expected_result = num_batches * batch_size / ( num_batches * batch_size - 1) * tfp.stats.covariance( x=np.reshape(x, [num_batches * batch_size, dim]), y=np.reshape(y, [num_batches * batch_size, dim])) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) for i in range(num_batches): sess.run(update_op, feed_dict={ placeholder_x: x[i], placeholder_y: y[i] }) result = sess.run(value) self.assertAllClose(expected_result, result, rtol=1e-15, atol=1e-15)
def test_covariance_random_singlet_batches_float32(self): from tensorflow_gan.python.eval import eval_utils """Checks value consistency of streaming_covariance.""" if tf.executing_eagerly(): # tf.placeholder() is not compatible with eager execution. return np.random.seed(0) neg = 8 data = np.random.randn(neg,2) placeholder = tf.compat.v1.placeholder(dtype=tf.float64, shape=(1,2)) value, update_op = eval_utils.streaming_covariance(x=placeholder) expected_result = np.cov(data, rowvar=False) with self.cached_session() as sess: sess.run(tf.compat.v1.initializers.local_variables()) for i in range(neg): update_op_result = sess.run(update_op, feed_dict={placeholder: [data[i]]}) result = sess.run(value) self.assertAllClose(update_op_result, result) # overall # pdb.set_trace() self.assertAllClose(expected_result, result)
def _frechet_classifier_distance_from_activations_helper( activations1, activations2, streaming=False): """A helper function evaluating the frechet classifier distance.""" activations1 = tf.convert_to_tensor(value=activations1) activations1.shape.assert_has_rank(2) activations2 = tf.convert_to_tensor(value=activations2) activations2.shape.assert_has_rank(2) activations_dtype = activations1.dtype if activations_dtype != tf.float64: activations1 = tf.cast(activations1, tf.float64) activations2 = tf.cast(activations2, tf.float64) # Compute mean and covariance matrices of activations. if streaming: m = eval_utils.streaming_mean_tensor_float64( tf.reduce_mean(input_tensor=activations1, axis=0)) m_w = eval_utils.streaming_mean_tensor_float64( tf.reduce_mean(input_tensor=activations2, axis=0)) sigma = eval_utils.streaming_covariance(activations1) sigma_w = eval_utils.streaming_covariance(activations2) else: m = (tf.reduce_mean(input_tensor=activations1, axis=0),) m_w = (tf.reduce_mean(input_tensor=activations2, axis=0),) # Calculate the unbiased covariance matrix of first activations. num_examples_real = tf.cast(tf.shape(input=activations1)[0], tf.float64) sigma = (num_examples_real / (num_examples_real - 1) * tfp.stats.covariance(activations1),) # Calculate the unbiased covariance matrix of second activations. num_examples_generated = tf.cast( tf.shape(input=activations2)[0], tf.float64) sigma_w = (num_examples_generated / (num_examples_generated - 1) * tfp.stats.covariance(activations2),) # m, m_w, sigma, sigma_w are tuples containing one or two elements: the first # element will be used to calculate the score value and the second will be # used to create the update_op. We apply the same operation on the two # elements to make sure their value is consistent. def _calculate_fid(m, m_w, sigma, sigma_w): """Returns the Frechet distance given the sample mean and covariance.""" # Find the Tr(sqrt(sigma sigma_w)) component of FID sqrt_trace_component = trace_sqrt_product(sigma, sigma_w) # Compute the two components of FID. # First the covariance component. # Here, note that trace(A + B) = trace(A) + trace(B) trace = tf.linalg.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component # Next the distance between means. mean = tf.reduce_sum(input_tensor=tf.math.squared_difference( m, m_w)) # Equivalent to L2 but more stable. fid = trace + mean if activations_dtype != tf.float64: fid = tf.cast(fid, activations_dtype) return fid result = tuple( _calculate_fid(m_val, m_w_val, sigma_val, sigma_w_val) for m_val, m_w_val, sigma_val, sigma_w_val in zip(m, m_w, sigma, sigma_w)) if streaming: return result else: return result[0]