def test_end2end(self, model_name, mock_get_input_fn_from_dataset):
"""End-to-end test of model_eval."""
tf_test_utils.write_fake_checkpoint('inception_v3',
self.test_session(),
self.checkpoint_dir,
FLAGS.moving_average_decay)
# Start up eval, loading that checkpoint.
FLAGS.batch_size = 2
FLAGS.checkpoint_dir = self.checkpoint_dir
FLAGS.eval_name = self.eval_name
FLAGS.max_evaluations = 1
FLAGS.max_examples = 2
FLAGS.best_checkpoint_metric = 'F1/All'
FLAGS.model_name = model_name
FLAGS.dataset_config_pbtxt = '/path/to/mock.pbtxt'
FLAGS.master = ''
# Always try to read in compressed inputs to stress that case. Uncompressed
# inputs are certain to work. This test is expensive to run, so we want to
# minimize the number of times we need to run this.
mock_get_input_fn_from_dataset.return_value = (
data_providers_test.make_golden_dataset(compressed_inputs=True,
use_tpu=FLAGS.use_tpu))
model_eval.main(0)
mock_get_input_fn_from_dataset.assert_called_once_with(
dataset_config_filename=FLAGS.dataset_config_pbtxt,
mode=tf.estimator.ModeKeys.EVAL,
use_tpu=FLAGS.use_tpu)
self.assertTrue(
tf_test_utils.check_file_exists('best_checkpoint.txt',
eval_name=self.eval_name))
self.assertTrue(
tf_test_utils.check_file_exists('best_checkpoint.metrics',
eval_name=self.eval_name))
def test_end2end_inception_v3_warm_up_from(self):
"""End-to-end test of model_train script."""
checkpoint_dir = tf_test_utils.test_tmpdir('inception_v3_warm_up_from')
tf_test_utils.write_fake_checkpoint('inception_v3', self.test_session(),
checkpoint_dir)
self._run_tiny_training(
model_name='inception_v3',
dataset=data_providers_test.make_golden_dataset(use_tpu=FLAGS.use_tpu),
warm_start_from=checkpoint_dir + '/model')
def test_end2end_inception_v3_warm_up_allow_different_num_channels(self):
"""End-to-end test of model_train script."""
FLAGS.allow_warmstart_from_different_num_channels = True
checkpoint_dir = tf_test_utils.test_tmpdir(
'inception_v3_warm_up_allow_different_num_channels')
tf_test_utils.write_fake_checkpoint(
'inception_v3',
self.test_session(),
checkpoint_dir,
num_channels=dv_constants.PILEUP_NUM_CHANNELS + 1)
self._run_tiny_training(
model_name='inception_v3',
dataset=data_providers_test.make_golden_dataset(use_tpu=FLAGS.use_tpu),
warm_start_from=checkpoint_dir + '/model')
def test_end2end_inception_v3_warm_up_from_mobilenet_v1(self):
"""Tests the behavior when warm start from mobilenet but train inception."""
checkpoint_dir = tf_test_utils.test_tmpdir(
'inception_v3_warm_up_from_mobilenet_v1')
tf_test_utils.write_fake_checkpoint('mobilenet_v1', self.test_session(),
checkpoint_dir)
self.assertTrue(
tf_test_utils.check_equals_checkpoint_top_scopes(
checkpoint_dir + '/model', ['MobilenetV1', 'global_step']))
self._run_tiny_training(
model_name='inception_v3',
dataset=data_providers_test.make_golden_dataset(use_tpu=FLAGS.use_tpu),
warm_start_from=checkpoint_dir + '/model')
self.assertTrue(
tf_test_utils.check_equals_checkpoint_top_scopes(
FLAGS.train_dir + '/model.ckpt-1', ['InceptionV3', 'global_step']))
def test_end2end_inception_v3_warm_up_by_default_fail_diff_num_channels(self):
"""End-to-end test of model_train script."""
checkpoint_dir = tf_test_utils.test_tmpdir(
'test_end2end_inception_v3_warm_up_by_default_fail_diff_num_channels')
tf_test_utils.write_fake_checkpoint(
'inception_v3',
self.test_session(),
checkpoint_dir,
num_channels=dv_constants.PILEUP_NUM_CHANNELS + 1)
with self.assertRaisesRegex(
ValueError,
r'Shape of variable InceptionV3/Conv2d_1a_3x3/weights:0 \(\(.*\)\) '
r'doesn\'t match with shape of tensor '
r'InceptionV3/Conv2d_1a_3x3/weights \(\[.*\]\) from checkpoint reader.'
):
self._run_tiny_training(
model_name='inception_v3',
dataset=data_providers_test.make_golden_dataset(
use_tpu=FLAGS.use_tpu),
warm_start_from=checkpoint_dir + '/model')
def test_fixed_eval_sees_the_same_evals(self,
mock_get_input_fn_from_dataset,
mock_checkpoints_iterator):
dataset = data_providers_test.make_golden_dataset(
use_tpu=FLAGS.use_tpu)
n_checkpoints = 3
checkpoints = [
tf_test_utils.write_fake_checkpoint('constant',
self.test_session(),
self.checkpoint_dir,
FLAGS.moving_average_decay,
name='model' + str(i))
for i in range(n_checkpoints)
]
# Setup our mocks.
mock_checkpoints_iterator.return_value = checkpoints
mock_get_input_fn_from_dataset.return_value = dataset
# Start up eval, loading that checkpoint.
FLAGS.batch_size = 2
FLAGS.checkpoint_dir = self.checkpoint_dir
FLAGS.eval_name = self.eval_name
FLAGS.max_evaluations = n_checkpoints
FLAGS.model_name = 'constant'
FLAGS.dataset_config_pbtxt = '/path/to/mock.pbtxt'
FLAGS.master = ''
model_eval.main(0)
self.assertEqual(mock_get_input_fn_from_dataset.call_args_list, [
mock.call(use_tpu=FLAGS.use_tpu,
dataset_config_filename=FLAGS.dataset_config_pbtxt,
mode=tf.estimator.ModeKeys.EVAL)
])
metrics = [
model_eval.read_metrics(checkpoint, eval_name=FLAGS.eval_name)
for checkpoint in checkpoints
]
# Check that our metrics are what we expect them to be.
# See b/62864044 for details on how to compute these counts:
# Counts of labels in our golden dataset:
# 1 0
# 12 1
# 35 2
expected_values_for_all_exact = {
# We have 12 correct calls [there are 12 variants with a label of 1] and
# 1 label 0 + 35 with a label of 2, so we have an accuracy of 12 / 48,
# which is 0.25.
'Accuracy/All': 0.25,
# We don't have any FNs because we call everything het.
'FNs/All': 0,
# Two of our labels are 0, which we call het, giving us 2 FP.
'FPs/All': 1.0,
# We call everything as het, so the recall has to be 1.
'Recall/All': 1.0,
# redacted
# # We don't call anything but hets, so TNs has to be 0.
# 'TNs/All': 0,
# We find 47 positives, so this has to be 47.
'TPs/All': 47,
}
for key, expected_value in expected_values_for_all_exact.iteritems():
print(str(key) + '=' + str(metrics[0][key]))
for key, expected_value in expected_values_for_all_exact.iteritems():
self.assertEqual(metrics[0][key], expected_value)
expected_values_for_all_close = {
# We called 47 / 48 correctly ~ 0.979167
'Precision/All': 0.979167,
# We called (2 * 47 / 48) / (1 + 47 / 48) correctly ~ 0.989474
'F1/All': 0.989474,
}
for key, expected_value in expected_values_for_all_close.iteritems():
self.assertAlmostEqual(metrics[0][key], expected_value, places=6)
for m1, m2 in zip(metrics, metrics[1:]):
self.assertEqual(m1, m2)
