def test_covariance_accuracy(self):
"""Checks accuracy of streaming_covariance."""
if tf.executing_eagerly():
# tf.placeholder() is not compatible with eager execution.
return
np.random.seed(0)
num_batches = 128
batch_size = 32
dim = 32
data = np.random.randn(num_batches, batch_size, dim)
placeholder = tf.compat.v1.placeholder(dtype=tf.float64,
shape=(batch_size, dim))
value, update_op = eval_utils.streaming_covariance(placeholder)
expected_result = np.cov(np.reshape(data,
[num_batches * batch_size, dim]),
rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
for i in range(num_batches):
sess.run(update_op, feed_dict={placeholder: data[i]})
result = sess.run(value)
self.assertAllClose(expected_result,
result,
rtol=1e-15,
atol=1e-15)
def test_covariance_float32(self):
"""Checks handling of float32 values in streaming_covariance."""
x = [[1., 2.], [2., 1.]]
result, update_op = eval_utils.streaming_covariance(
x=tf.constant(x, dtype=tf.float32))
expected_result = np.cov(x, rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
self.assertAllClose(expected_result, update_op)
self.assertAllClose(expected_result, result)
def test_covariance_simple(self):
"""Sanity check for streaming_covariance."""
if tf.executing_eagerly():
# streaming_covariance is not supported when eager execution is enabled.
return
x = [[1., 2.], [2., 1.]]
result, update_op = eval_utils.streaming_covariance(
tf.constant(x, dtype=tf.float64))
expected_result = np.cov(x, rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
self.assertAllClose(expected_result, update_op)
self.assertAllClose(expected_result, result)
def test_covariance_float32(self):
from tensorflow_gan.python.eval import eval_utils
"""Checks handling of float32 values in streaming_covariance."""
if tf.executing_eagerly():
# streaming_covariance is not supported when eager execution is enabled.
return
x = [[1., 2.], [2., 1.]]
result, update_op = eval_utils.streaming_covariance(
x=tf.constant(x, dtype=tf.float32))
expected_result = np.cov(x, rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
self.assertAllClose(expected_result, update_op)
self.assertAllClose(expected_result, result)
def test_covariance_float32_with_y(self):
"""Checks handling of float32 values in streaming_covariance."""
x = [[1., 2.], [2., 1.]]
y = [[1., 2.], [2., 1.]]
result, update_op = eval_utils.streaming_covariance(
x=tf.constant(x, dtype=tf.float32),
y=tf.constant(y, dtype=tf.float32))
# We mulitiply by N/(N-1)=2 to get the unbiased estimator.
# Note that we cannot use np.cov since it uses a different semantics for y.
expected_result = 2. * tfp.stats.covariance(x, y)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
self.assertAllClose(expected_result, update_op)
self.assertAllClose(expected_result, result)
def test_covariance_with_y(self):
"""Checks output of streaming_covariance given value for y."""
if tf.executing_eagerly():
# streaming_covariance is not supported when eager execution is enabled.
return
x = [[1., 2.], [2., 1.]]
y = [[3., 3.], [1., 0.]]
result, update_op = eval_utils.streaming_covariance(
x=tf.constant(x, dtype=tf.float64),
y=tf.constant(y, dtype=tf.float64))
# We mulitiply by N/(N-1)=2 to get the unbiased estimator.
# Note that we cannot use np.cov since it uses a different semantics for y.
expected_result = 2. * tfp.stats.covariance(x, y)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
self.assertAllClose(expected_result, update_op)
self.assertAllClose(expected_result, result)
def test_covariance_batches(self):
"""Checks value consistency of streaming_covariance."""
np.random.seed(0)
num_batches = 8
data = np.random.randn(num_batches, 4, 5)
placeholder = tf.compat.v1.placeholder(dtype=tf.float64, shape=(4, 5))
value, update_op = eval_utils.streaming_covariance(placeholder)
expected_result = np.cov(np.reshape(data, [num_batches * 4, 5]),
rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
for i in range(num_batches):
update_op_result = sess.run(update_op,
feed_dict={placeholder: data[i]})
result = sess.run(value)
self.assertAllClose(update_op_result, result)
self.assertAllClose(expected_result, result)
def test_covariance_accuracy_with_y(self):
"""Checks accuracy of streaming_covariance with two input tensors."""
if tf.executing_eagerly():
# tf.placeholder() is not compatible with eager execution.
return
np.random.seed(0)
num_batches = 128
batch_size = 32
dim = 32
x = np.random.randn(num_batches, batch_size, dim)
y = np.random.randn(num_batches, batch_size, dim)
placeholder_x = tf.compat.v1.placeholder(dtype=tf.float64,
shape=(batch_size, dim))
placeholder_y = tf.compat.v1.placeholder(dtype=tf.float64,
shape=(batch_size, dim))
value, update_op = eval_utils.streaming_covariance(
placeholder_x, placeholder_y)
# We mulitiply by N/(N-1) to get the unbiased estimator.
# Note that we cannot use np.cov since it uses different semantics for y.
expected_result = num_batches * batch_size / (
num_batches * batch_size - 1) * tfp.stats.covariance(
x=np.reshape(x, [num_batches * batch_size, dim]),
y=np.reshape(y, [num_batches * batch_size, dim]))
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
for i in range(num_batches):
sess.run(update_op,
feed_dict={
placeholder_x: x[i],
placeholder_y: y[i]
})
result = sess.run(value)
self.assertAllClose(expected_result,
result,
rtol=1e-15,
atol=1e-15)
def test_covariance_random_singlet_batches_float32(self):
from tensorflow_gan.python.eval import eval_utils
"""Checks value consistency of streaming_covariance."""
if tf.executing_eagerly():
# tf.placeholder() is not compatible with eager execution.
return
np.random.seed(0)
neg = 8
data = np.random.randn(neg,2)
placeholder = tf.compat.v1.placeholder(dtype=tf.float64, shape=(1,2))
value, update_op = eval_utils.streaming_covariance(x=placeholder)
expected_result = np.cov(data, rowvar=False)
with self.cached_session() as sess:
sess.run(tf.compat.v1.initializers.local_variables())
for i in range(neg):
update_op_result = sess.run(update_op, feed_dict={placeholder: [data[i]]})
result = sess.run(value)
self.assertAllClose(update_op_result, result)
# overall
# pdb.set_trace()
self.assertAllClose(expected_result, result)
def _frechet_classifier_distance_from_activations_helper(
activations1, activations2, streaming=False):
"""A helper function evaluating the frechet classifier distance."""
activations1 = tf.convert_to_tensor(value=activations1)
activations1.shape.assert_has_rank(2)
activations2 = tf.convert_to_tensor(value=activations2)
activations2.shape.assert_has_rank(2)
activations_dtype = activations1.dtype
if activations_dtype != tf.float64:
activations1 = tf.cast(activations1, tf.float64)
activations2 = tf.cast(activations2, tf.float64)
# Compute mean and covariance matrices of activations.
if streaming:
m = eval_utils.streaming_mean_tensor_float64(
tf.reduce_mean(input_tensor=activations1, axis=0))
m_w = eval_utils.streaming_mean_tensor_float64(
tf.reduce_mean(input_tensor=activations2, axis=0))
sigma = eval_utils.streaming_covariance(activations1)
sigma_w = eval_utils.streaming_covariance(activations2)
else:
m = (tf.reduce_mean(input_tensor=activations1, axis=0),)
m_w = (tf.reduce_mean(input_tensor=activations2, axis=0),)
# Calculate the unbiased covariance matrix of first activations.
num_examples_real = tf.cast(tf.shape(input=activations1)[0], tf.float64)
sigma = (num_examples_real / (num_examples_real - 1) *
tfp.stats.covariance(activations1),)
# Calculate the unbiased covariance matrix of second activations.
num_examples_generated = tf.cast(
tf.shape(input=activations2)[0], tf.float64)
sigma_w = (num_examples_generated / (num_examples_generated - 1) *
tfp.stats.covariance(activations2),)
# m, m_w, sigma, sigma_w are tuples containing one or two elements: the first
# element will be used to calculate the score value and the second will be
# used to create the update_op. We apply the same operation on the two
# elements to make sure their value is consistent.
def _calculate_fid(m, m_w, sigma, sigma_w):
"""Returns the Frechet distance given the sample mean and covariance."""
# Find the Tr(sqrt(sigma sigma_w)) component of FID
sqrt_trace_component = trace_sqrt_product(sigma, sigma_w)
# Compute the two components of FID.
# First the covariance component.
# Here, note that trace(A + B) = trace(A) + trace(B)
trace = tf.linalg.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component
# Next the distance between means.
mean = tf.reduce_sum(input_tensor=tf.math.squared_difference(
m, m_w)) # Equivalent to L2 but more stable.
fid = trace + mean
if activations_dtype != tf.float64:
fid = tf.cast(fid, activations_dtype)
return fid
result = tuple(
_calculate_fid(m_val, m_w_val, sigma_val, sigma_w_val)
for m_val, m_w_val, sigma_val, sigma_w_val in zip(m, m_w, sigma, sigma_w))
if streaming:
return result
else:
return result[0]
