def test_single_example_correct(self, mock_tfhub_load): mock_tfhub_load.return_value = fake_logit_fn real_score = util.mnist_score(real_digit()) fake_score = util.mnist_score(fake_digit()) with self.cached_session() as sess: self.assertNear(1.0, sess.run(real_score), 1e-6) self.assertNear(1.0, sess.run(fake_score), 1e-6)
def test_single_example_correct(self): if tf.executing_eagerly(): # `run_image_classifier` doesn't work in eager. return real_score = util.mnist_score(real_digit()) fake_score = util.mnist_score(fake_digit()) with self.cached_session() as sess: self.assertNear(1.0, sess.run(real_score), 1e-6) self.assertNear(1.0, sess.run(fake_score), 1e-6)
def get_metrics(gan_model): """Return metrics for MNIST experiment.""" real_mnist_score = util.mnist_score(gan_model.real_data) generated_mnist_score = util.mnist_score(gan_model.generated_data) frechet_distance = util.mnist_frechet_distance(gan_model.real_data, gan_model.generated_data) return { 'real_mnist_score': tf.compat.v1.metrics.mean(real_mnist_score), 'mnist_score': tf.compat.v1.metrics.mean(generated_mnist_score), 'frechet_distance': tf.compat.v1.metrics.mean(frechet_distance), }
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ # Fetch real images. with tf.compat.v1.name_scope('inputs'): real_images, _ = data_provider.provide_data( 'train', hparams.num_images_generated, hparams.dataset_dir) image_write_ops = None if hparams.eval_real_images: tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(real_images, hparams.classifier_filename)) else: # In order for variables to load, use the same variable scope as in the # train job. with tf.compat.v1.variable_scope('Generator'): images = networks.unconditional_generator(tf.random.normal( [hparams.num_images_generated, hparams.noise_dims]), is_training=False) tf.compat.v1.summary.scalar( 'MNIST_Frechet_distance', util.mnist_frechet_distance(real_images, images, hparams.classifier_filename)) tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(images, hparams.classifier_filename)) if hparams.num_images_generated >= 100 and hparams.write_to_disk: reshaped_images = tfgan.eval.image_reshaper(images[:100, ...], num_cols=10) uint8_images = data_provider.float_image_to_uint8(reshaped_images) image_write_ops = tf.io.write_file( '%s/%s' % (hparams.eval_dir, 'unconditional_gan.png'), tf.image.encode_png(uint8_images[0])) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def _disabled_test_minibatch_correct(self): """Tests the correctness of the mnist_score function.""" # Disabled since it requires loading the tfhub MNIST module. mscore = util.mnist_score( tf.concat([real_digit(), real_digit(), fake_digit()], 0)) with self.cached_session() as sess: self.assertNear(1.612828, sess.run(mscore), 1e-6)
def test_batch_splitting_doesnt_change_value(self): for num_batches in [1, 2, 4, 8]: mscore = util.mnist_score(tf.concat([real_digit()] * 4 + [fake_digit()] * 4, 0), num_batches=num_batches) with self.cached_session() as sess: self.assertNear(1.649209, sess.run(mscore), 1e-6)
def test_minibatch_correct(self): if tf.executing_eagerly(): # `run_image_classifier` doesn't work in eager. return mscore = util.mnist_score( tf.concat([real_digit(), real_digit(), fake_digit()], 0)) with self.cached_session() as sess: self.assertNear(1.612828, sess.run(mscore), 1e-6)
def _disabled_test_batch_splitting_doesnt_change_value(self): """Tests the correctness of mnist_score function over different batches.""" # Disabled since it requires loading the tfhub MNIST module. for num_batches in [1, 2, 4, 8]: mscore = util.mnist_score(tf.concat([real_digit()] * 4 + [fake_digit()] * 4, 0), num_batches=num_batches) with self.cached_session() as sess: self.assertNear(1.649209, sess.run(mscore), 1e-6)
def test_any_batch_size(self): if tf.executing_eagerly(): # Placeholders don't work in eager execution mode. return inputs = tf.compat.v1.placeholder(tf.float32, shape=[None, 28, 28, 1]) mscore = util.mnist_score(inputs) for batch_size in [4, 16, 30]: with self.cached_session() as sess: sess.run(mscore, feed_dict={inputs: np.zeros([batch_size, 28, 28, 1])})
def test_any_batch_size(self, mock_tfhub_load): mock_tfhub_load.return_value = fake_logit_fn # Create a graph since placeholders don't work in eager execution mode. with tf.Graph().as_default(): inputs = tf.placeholder(tf.float32, shape=[None, 28, 28, 1]) mscore = util.mnist_score(inputs) for batch_size in [4, 16, 30]: with self.cached_session() as sess: sess.run( mscore, feed_dict={inputs: np.zeros([batch_size, 28, 28, 1])})
def test_batch_splitting_doesnt_change_value(self): if tf.executing_eagerly(): # `run_image_classifier` doesn't work in eager. return for num_batches in [1, 2, 4, 8]: mscore = util.mnist_score( tf.concat([real_digit()] * 4 + [fake_digit()] * 4, 0), num_batches=num_batches) with self.cached_session() as sess: self.assertNear(1.649209, sess.run(mscore), 1e-6)
def test_deterministic(self): m_score = util.mnist_score(real_digit()) with self.cached_session() as sess: m_score1 = sess.run(m_score) m_score2 = sess.run(m_score) self.assertEqual(m_score1, m_score2) with self.cached_session() as sess: m_score3 = sess.run(m_score) self.assertEqual(m_score1, m_score3)
def test_deterministic(self, mock_tfhub_load): mock_tfhub_load.return_value = fake_logit_fn m_score = util.mnist_score(real_digit()) with self.cached_session() as sess: m_score1 = sess.run(m_score) m_score2 = sess.run(m_score) self.assertEqual(m_score1, m_score2) with self.cached_session() as sess: m_score3 = sess.run(m_score) self.assertEqual(m_score1, m_score3)
def test_deterministic(self): if tf.executing_eagerly(): # `run_image_classifier` doesn't work in eager. return m_score = util.mnist_score(real_digit()) with self.cached_session() as sess: m_score1 = sess.run(m_score) m_score2 = sess.run(m_score) self.assertEqual(m_score1, m_score2) with self.cached_session() as sess: m_score3 = sess.run(m_score) self.assertEqual(m_score1, m_score3)
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ with tf.compat.v1.name_scope('inputs'): noise, one_hot_labels = _get_generator_inputs( hparams.num_images_per_class, NUM_CLASSES, hparams.noise_dims) # Generate images. with tf.compat.v1.variable_scope( 'Generator'): # Same scope as in train job. images = networks.conditional_generator((noise, one_hot_labels), is_training=False) # Visualize images. reshaped_img = tfgan.eval.image_reshaper( images, num_cols=hparams.num_images_per_class) tf.compat.v1.summary.image('generated_images', reshaped_img, max_outputs=1) # Calculate evaluation metrics. tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(images, hparams.classifier_filename)) tf.compat.v1.summary.scalar( 'MNIST_Cross_entropy', util.mnist_cross_entropy(images, one_hot_labels, hparams.classifier_filename)) # Write images to disk. image_write_ops = None if hparams.write_to_disk: image_write_ops = tf.io.write_file( '%s/%s' % (hparams.eval_dir, 'conditional_gan.png'), tf.image.encode_png( data_provider.float_image_to_uint8(reshaped_img[0]))) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ with tf.name_scope('inputs'): noise_args = (hparams.noise_samples, CAT_SAMPLE_POINTS, CONT_SAMPLE_POINTS, hparams.unstructured_noise_dims, hparams.continuous_noise_dims) # Use fixed noise vectors to illustrate the effect of each dimension. display_noise1 = util.get_eval_noise_categorical(*noise_args) display_noise2 = util.get_eval_noise_continuous_dim1(*noise_args) display_noise3 = util.get_eval_noise_continuous_dim2(*noise_args) _validate_noises([display_noise1, display_noise2, display_noise3]) # Visualize the effect of each structured noise dimension on the generated # image. def generator_fn(inputs): return networks.infogan_generator(inputs, len(CAT_SAMPLE_POINTS), is_training=False) with tf.variable_scope( 'Generator') as genscope: # Same scope as in training. categorical_images = generator_fn(display_noise1) reshaped_categorical_img = tfgan.eval.image_reshaper( categorical_images, num_cols=len(CAT_SAMPLE_POINTS)) tf.summary.image('categorical', reshaped_categorical_img, max_outputs=1) with tf.variable_scope(genscope, reuse=True): continuous1_images = generator_fn(display_noise2) reshaped_continuous1_img = tfgan.eval.image_reshaper( continuous1_images, num_cols=len(CONT_SAMPLE_POINTS)) tf.summary.image('continuous1', reshaped_continuous1_img, max_outputs=1) with tf.variable_scope(genscope, reuse=True): continuous2_images = generator_fn(display_noise3) reshaped_continuous2_img = tfgan.eval.image_reshaper( continuous2_images, num_cols=len(CONT_SAMPLE_POINTS)) tf.summary.image('continuous2', reshaped_continuous2_img, max_outputs=1) # Evaluate image quality. all_images = tf.concat( [categorical_images, continuous1_images, continuous2_images], 0) tf.summary.scalar('MNIST_Classifier_score', util.mnist_score(all_images)) # Write images to disk. image_write_ops = [] if hparams.write_to_disk: image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'categorical_infogan.png', reshaped_categorical_img[0])) image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'continuous1_infogan.png', reshaped_continuous1_img[0])) image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'continuous2_infogan.png', reshaped_continuous2_img[0])) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def test_single_example_correct(self): real_score = util.mnist_score(real_digit()) fake_score = util.mnist_score(fake_digit()) with self.cached_session() as sess: self.assertNear(1.0, sess.run(real_score), 1e-6) self.assertNear(1.0, sess.run(fake_score), 1e-6)
def test_minibatch_correct(self): mscore = util.mnist_score( tf.concat([real_digit(), real_digit(), fake_digit()], 0)) with self.cached_session() as sess: self.assertNear(1.612828, sess.run(mscore), 1e-6)
def get_inception_score(images, splits=10): images = tf.convert_to_tensor(images) generated_mnist_score = eval_util.mnist_score(images) print(generated_mnist_score) return [generated_mnist_score]
def get_mnist_eval_metrics(real, fake): frechet = tfgan_mnist.mnist_frechet_distance(real, fake, 1) score = tfgan_mnist.mnist_score(fake, 1) return tf.stack(list(map(tf.stop_gradient, (frechet, score))))