def testEvaluateWithEvalFeedDict(self): if tf.executing_eagerly(): # tf.placeholder() is not compatible with eager execution. return # Create a checkpoint. checkpoint_dir = os.path.join(self.get_temp_dir(), 'evaluate_with_eval_feed_dict') self._train_model(checkpoint_dir, num_steps=1) # We need a variable that the saver will try to restore. tf.compat.v1.train.get_or_create_global_step() # Create a variable and an eval op that increments it with a placeholder. try: my_var = _local_variable(0.0, name='my_var') except TypeError: # `collections` doesn't exist in TF2. return increment = tf.compat.v1.placeholder(dtype=tf.float32) eval_ops = tf.compat.v1.assign_add(my_var, increment) increment_value = 3 num_evals = 5 expected_value = increment_value * num_evals final_values = evaluation.evaluate_repeatedly( checkpoint_dir=checkpoint_dir, eval_ops=eval_ops, feed_dict={increment: 3}, final_ops={'my_var': tf.identity(my_var)}, hooks=[ evaluation.StopAfterNEvalsHook(num_evals), ], max_number_of_evaluations=1) self.assertEqual(final_values['my_var'], expected_value)
def testEvaluatePerfectModel(self): if tf.executing_eagerly(): return checkpoint_dir = os.path.join(self.get_temp_dir(), 'evaluate_perfect_model_repeated') # Train a Model to completion: self._train_model(checkpoint_dir, num_steps=300) # Run inputs = tf.constant(self._inputs, dtype=tf.float32) labels = tf.constant(self._labels, dtype=tf.float32) logits = logistic_classifier(inputs) predictions = tf.round(logits) accuracy, update_op = tf.compat.v1.metrics.accuracy( predictions=predictions, labels=labels) final_values = evaluation.evaluate_repeatedly( checkpoint_dir=checkpoint_dir, eval_ops=update_op, final_ops={'accuracy': accuracy}, hooks=[ evaluation.StopAfterNEvalsHook(1), ], max_number_of_evaluations=1) self.assertTrue(final_values['accuracy'] > .99)
def testEvaluationLoopTimeout(self): if tf.executing_eagerly(): # This test uses `tf.placeholder`, which doesn't work in eager executing. return checkpoint_dir = os.path.join(self.get_temp_dir(), 'evaluation_loop_timeout') if not tf.io.gfile.exists(checkpoint_dir): tf.io.gfile.makedirs(checkpoint_dir) # We need a variable that the saver will try to restore. tf.compat.v1.train.get_or_create_global_step() # Run with placeholders. If we actually try to evaluate this, we'd fail # since we're not using a feed_dict. cant_run_op = tf.compat.v1.placeholder(dtype=tf.float32) start = time.time() final_values = evaluation.evaluate_repeatedly( checkpoint_dir=checkpoint_dir, eval_ops=cant_run_op, hooks=[evaluation.StopAfterNEvalsHook(10)], timeout=6) end = time.time() self.assertFalse(final_values) # Assert that we've waited for the duration of the timeout (minus the sleep # time). self.assertGreater(end - start, 5.0) # Then the timeout kicked in and stops the loop. self.assertLess(end - start, 7)
def testEvalOpAndFinalOp(self): if tf.executing_eagerly(): return checkpoint_dir = os.path.join(self.get_temp_dir(), 'eval_ops_and_final_ops') # Train a model for a single step to get a checkpoint. self._train_model(checkpoint_dir, num_steps=1) checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) # Create the model so we have something to restore. inputs = tf.constant(self._inputs, dtype=tf.float32) logistic_classifier(inputs) num_evals = 5 final_increment = 9.0 try: my_var = _local_variable(0.0, name='MyVar') except TypeError: # `collections` doesn't exist in TF2. return eval_ops = tf.compat.v1.assign_add(my_var, 1.0) final_ops = tf.identity(my_var) + final_increment final_ops_values = evaluation.evaluate_once( checkpoint_path=checkpoint_path, eval_ops=eval_ops, final_ops={'value': final_ops}, hooks=[ evaluation.StopAfterNEvalsHook(num_evals), ]) self.assertEqual(final_ops_values['value'], num_evals + final_increment)
def testEvaluatePerfectModel(self): if tf.executing_eagerly(): # tf.metrics.accuracy is not supported when eager execution is enabled. return checkpoint_dir = os.path.join(self.get_temp_dir(), 'evaluate_perfect_model_once') # Train a Model to completion: self._train_model(checkpoint_dir, num_steps=300) # Run inputs = tf.constant(self._inputs, dtype=tf.float32) labels = tf.constant(self._labels, dtype=tf.float32) logits = logistic_classifier(inputs) predictions = tf.round(logits) accuracy, update_op = tf.compat.v1.metrics.accuracy( predictions=predictions, labels=labels) checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir) final_ops_values = evaluation.evaluate_once( checkpoint_path=checkpoint_path, eval_ops=update_op, final_ops={'accuracy': accuracy}, hooks=[ evaluation.StopAfterNEvalsHook(1), ]) self.assertTrue(final_ops_values['accuracy'] > .99)
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ # Fetch and generate images to run through Inception. with tf.name_scope('inputs'): real_data, _ = data_provider.provide_data('test', hparams.num_images_generated, shuffle=False) generated_data = _get_generated_data(hparams.num_images_generated) # Compute Frechet Inception Distance. if hparams.eval_frechet_inception_distance: fid = util.get_frechet_inception_distance(real_data, generated_data, hparams.num_images_generated, hparams.num_inception_images) tf.summary.scalar('frechet_inception_distance', fid) # Compute normal Inception scores. if hparams.eval_real_images: inc_score = util.get_inception_scores(real_data, hparams.num_images_generated, hparams.num_inception_images) else: inc_score = util.get_inception_scores(generated_data, hparams.num_images_generated, hparams.num_inception_images) tf.summary.scalar('inception_score', inc_score) # Create ops that write images to disk. image_write_ops = None if hparams.num_images_generated >= 100 and hparams.write_to_disk: reshaped_imgs = tfgan.eval.image_reshaper(generated_data[:100], num_cols=10) uint8_images = data_provider.float_image_to_uint8(reshaped_imgs) image_write_ops = tf.io.write_file( '%s/%s' % (hparams.eval_dir, 'unconditional_cifar10.png'), tf.image.encode_png(uint8_images[0])) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, master=hparams.master, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ with tf.compat.v1.name_scope('inputs'): noise, one_hot_labels = _get_generator_inputs( hparams.num_images_per_class, NUM_CLASSES, hparams.noise_dims) # Generate images. with tf.compat.v1.variable_scope( 'Generator'): # Same scope as in train job. images = networks.conditional_generator((noise, one_hot_labels), is_training=False) # Visualize images. reshaped_img = tfgan.eval.image_reshaper( images, num_cols=hparams.num_images_per_class) tf.compat.v1.summary.image('generated_images', reshaped_img, max_outputs=1) # Calculate evaluation metrics. tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(images, hparams.classifier_filename)) tf.compat.v1.summary.scalar( 'MNIST_Cross_entropy', util.mnist_cross_entropy(images, one_hot_labels, hparams.classifier_filename)) # Write images to disk. image_write_ops = None if hparams.write_to_disk: image_write_ops = tf.io.write_file( '%s/%s' % (hparams.eval_dir, 'conditional_gan.png'), tf.image.encode_png( data_provider.float_image_to_uint8(reshaped_img[0]))) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ # Fetch real images. with tf.compat.v1.name_scope('inputs'): real_images, _ = data_provider.provide_data( 'train', hparams.num_images_generated, hparams.dataset_dir) image_write_ops = None if hparams.eval_real_images: tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(real_images, hparams.classifier_filename)) else: # In order for variables to load, use the same variable scope as in the # train job. with tf.compat.v1.variable_scope('Generator'): images = networks.unconditional_generator(tf.random.normal( [hparams.num_images_generated, hparams.noise_dims]), is_training=False) tf.compat.v1.summary.scalar( 'MNIST_Frechet_distance', util.mnist_frechet_distance(real_images, images, hparams.classifier_filename)) tf.compat.v1.summary.scalar( 'MNIST_Classifier_score', util.mnist_score(images, hparams.classifier_filename)) if hparams.num_images_generated >= 100 and hparams.write_to_disk: reshaped_images = tfgan.eval.image_reshaper(images[:100, ...], num_cols=10) uint8_images = data_provider.float_image_to_uint8(reshaped_images) image_write_ops = tf.io.write_file( '%s/%s' % (hparams.eval_dir, 'unconditional_gan.png'), tf.image.encode_png(uint8_images[0])) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)
def testEvaluationLoopTimeoutWithTimeoutFn(self): if tf.executing_eagerly(): # tf.metrics.accuracy is not supported when eager execution is enabled. return checkpoint_dir = os.path.join( self.get_temp_dir(), 'evaluation_loop_timeout_with_timeout_fn') # Train a Model to completion: self._train_model(checkpoint_dir, num_steps=300) # Run inputs = tf.constant(self._inputs, dtype=tf.float32) labels = tf.constant(self._labels, dtype=tf.float32) logits = logistic_classifier(inputs) predictions = tf.round(logits) accuracy, update_op = tf.compat.v1.metrics.accuracy( predictions=predictions, labels=labels) timeout_fn_calls = [0] def timeout_fn(): timeout_fn_calls[0] += 1 return timeout_fn_calls[0] > 3 final_values = evaluation.evaluate_repeatedly( checkpoint_dir=checkpoint_dir, eval_ops=update_op, final_ops={'accuracy': accuracy}, hooks=[ evaluation.StopAfterNEvalsHook(1), ], eval_interval_secs=1, max_number_of_evaluations=2, timeout=0.1, timeout_fn=timeout_fn) # We should have evaluated once. self.assertTrue(final_values['accuracy'] > .99) # And called 4 times the timeout fn self.assertEqual(4, timeout_fn_calls[0])
def evaluate(hparams, run_eval_loop=True): """Runs an evaluation loop. Args: hparams: An HParams instance containing the eval hyperparameters. run_eval_loop: Whether to run the full eval loop. Set to False for testing. """ with tf.name_scope('inputs'): noise_args = (hparams.noise_samples, CAT_SAMPLE_POINTS, CONT_SAMPLE_POINTS, hparams.unstructured_noise_dims, hparams.continuous_noise_dims) # Use fixed noise vectors to illustrate the effect of each dimension. display_noise1 = util.get_eval_noise_categorical(*noise_args) display_noise2 = util.get_eval_noise_continuous_dim1(*noise_args) display_noise3 = util.get_eval_noise_continuous_dim2(*noise_args) _validate_noises([display_noise1, display_noise2, display_noise3]) # Visualize the effect of each structured noise dimension on the generated # image. def generator_fn(inputs): return networks.infogan_generator(inputs, len(CAT_SAMPLE_POINTS), is_training=False) with tf.variable_scope( 'Generator') as genscope: # Same scope as in training. categorical_images = generator_fn(display_noise1) reshaped_categorical_img = tfgan.eval.image_reshaper( categorical_images, num_cols=len(CAT_SAMPLE_POINTS)) tf.summary.image('categorical', reshaped_categorical_img, max_outputs=1) with tf.variable_scope(genscope, reuse=True): continuous1_images = generator_fn(display_noise2) reshaped_continuous1_img = tfgan.eval.image_reshaper( continuous1_images, num_cols=len(CONT_SAMPLE_POINTS)) tf.summary.image('continuous1', reshaped_continuous1_img, max_outputs=1) with tf.variable_scope(genscope, reuse=True): continuous2_images = generator_fn(display_noise3) reshaped_continuous2_img = tfgan.eval.image_reshaper( continuous2_images, num_cols=len(CONT_SAMPLE_POINTS)) tf.summary.image('continuous2', reshaped_continuous2_img, max_outputs=1) # Evaluate image quality. all_images = tf.concat( [categorical_images, continuous1_images, continuous2_images], 0) tf.summary.scalar('MNIST_Classifier_score', util.mnist_score(all_images)) # Write images to disk. image_write_ops = [] if hparams.write_to_disk: image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'categorical_infogan.png', reshaped_categorical_img[0])) image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'continuous1_infogan.png', reshaped_continuous1_img[0])) image_write_ops.append( _get_write_image_ops(hparams.eval_dir, 'continuous2_infogan.png', reshaped_continuous2_img[0])) # For unit testing, use `run_eval_loop=False`. if not run_eval_loop: return evaluation.evaluate_repeatedly( hparams.checkpoint_dir, hooks=[ evaluation.SummaryAtEndHook(hparams.eval_dir), evaluation.StopAfterNEvalsHook(1) ], eval_ops=image_write_ops, max_number_of_evaluations=hparams.max_number_of_evaluations)