def _create_names_to_metrics(self, predictions, labels):
accuracy0, update_op0 = metrics.accuracy(
labels=labels, predictions=predictions)
accuracy1, update_op1 = metrics.accuracy(
labels=labels, predictions=predictions + 1)
names_to_values = {'Accuracy': accuracy0, 'Another_accuracy': accuracy1}
names_to_updates = {'Accuracy': update_op0, 'Another_accuracy': update_op1}
return names_to_values, names_to_updates
def _ModelFn(features, labels, mode):
if is_training:
logits_out = self._BuildGraph(features)
else:
graph_def = self._GetGraphDef(use_trt, batch_size, model_dir)
logits_out = importer.import_graph_def(
graph_def,
input_map={INPUT_NODE_NAME: features},
return_elements=[OUTPUT_NODE_NAME + ':0'],
name='')[0]
loss = losses.sparse_softmax_cross_entropy(
labels=labels, logits=logits_out)
summary.scalar('loss', loss)
classes_out = math_ops.argmax(logits_out, axis=1, name='classes_out')
accuracy = metrics.accuracy(
labels=labels, predictions=classes_out, name='acc_op')
summary.scalar('accuracy', accuracy[1])
if mode == ModeKeys.EVAL:
return EstimatorSpec(
mode, loss=loss, eval_metric_ops={'accuracy': accuracy})
elif mode == ModeKeys.TRAIN:
optimizer = AdamOptimizer(learning_rate=1e-2)
train_op = optimizer.minimize(loss, global_step=get_global_step())
return EstimatorSpec(mode, loss=loss, train_op=train_op)
def testWithEpochLimit(self):
predictions_limited = input.limit_epochs(self._predictions, num_epochs=1)
labels_limited = input.limit_epochs(self._labels, num_epochs=1)
value_op, update_op = metrics.accuracy(
labels=labels_limited, predictions=predictions_limited)
init_op = control_flow_ops.group(variables.global_variables_initializer(),
variables.local_variables_initializer())
# Create checkpoint and log directories:
chkpt_dir = os.path.join(self.get_temp_dir(), 'tmp_logs/')
gfile.MakeDirs(chkpt_dir)
logdir = os.path.join(self.get_temp_dir(), 'tmp_logs2/')
gfile.MakeDirs(logdir)
# Save initialized variables to a checkpoint directory:
saver = saver_lib.Saver()
with self.test_session() as sess:
init_op.run()
saver.save(sess, os.path.join(chkpt_dir, 'chkpt'))
# Now, run the evaluation loop:
accuracy_value = evaluation.evaluation_loop(
'', chkpt_dir, logdir, eval_op=update_op, final_op=value_op,
max_number_of_evaluations=1, num_evals=10000)
self.assertAlmostEqual(accuracy_value, self._expected_accuracy)
def _eval_metric_ops(self, labels, class_ids, weights, weighted_sum_loss,
example_weight_sum):
"""Returns the Eval metric ops."""
with ops.name_scope(
None, 'metrics',
(labels, class_ids, weights, weighted_sum_loss, example_weight_sum)):
keys = metric_keys.MetricKeys
metric_ops = {
# Estimator already adds a metric for loss.
# TODO(xiejw): Any other metrics?
_summary_key(self._name, keys.LOSS_MEAN):
metrics_lib.mean(
# Both values and weights here are reduced, scalar Tensors.
# values is the actual mean we want -- weights represents the
# total weight of the batch and is needed to calculate
# update_op over many batches.
values=(weighted_sum_loss / example_weight_sum),
weights=example_weight_sum,
name=keys.LOSS_MEAN),
_summary_key(self._name, keys.ACCURACY):
metrics_lib.accuracy(
labels=labels,
predictions=class_ids,
weights=weights,
name=keys.ACCURACY),
}
return metric_ops
def model_fn(self, mode, features, labels, params):
c = variable_scope.get_variable(
'c',
initializer=constant_op.constant(10, dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = math_ops.multiply(features, c)
loss = None
if mode is not model_fn_lib.ModeKeys.PREDICT:
loss = losses.absolute_difference(
labels=labels,
predictions=predictions,
reduction=losses.Reduction.SUM)
loss = math_ops.reduce_sum(loss)
metrics = {
'accuracy': metrics_lib.accuracy(labels, predictions),
'auc': metrics_lib.auc(labels, predictions)
}
return model_fn_lib.EstimatorSpec(
mode=mode,
loss=loss,
eval_metric_ops=metrics,
predictions={'probabilities': predictions},
train_op=control_flow_ops.no_op()) # This train_op isn't actually used.
def testEvaluateWithFiniteInputs(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'evaluate_with_finite_inputs')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run evaluation. Inputs are fed through input producer for one epoch.
all_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
all_labels = constant_op.constant(self._labels, dtype=dtypes.float32)
single_input, single_label = training.slice_input_producer(
[all_inputs, all_labels], num_epochs=1)
inputs, labels = training.batch([single_input, single_label], batch_size=6,
allow_smaller_final_batch=True)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(
predictions=predictions, labels=labels)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=update_op,
final_ops={'accuracy': accuracy,
'eval_steps': evaluation._get_or_create_eval_step()},
hooks=[evaluation._StopAfterNEvalsHook(None),])
self.assertTrue(final_ops_values['accuracy'] > .99)
# Runs evaluation for 4 iterations. First 2 evaluate full batch of 6 inputs
# each; the 3rd iter evaluates the remaining 4 inputs, and the last one
# triggers an error which stops evaluation.
self.assertEqual(final_ops_values['eval_steps'], 4)
def testAdditionalHooks(self):
checkpoint_path = os.path.join(self.get_temp_dir(), 'model.ckpt')
log_dir = os.path.join(self.get_temp_dir(), 'log_dir1/')
# First, save out the current model to a checkpoint:
self._prepareCheckpoint(checkpoint_path)
# Next, determine the metric to evaluate:
value_op, update_op = metrics.accuracy(
labels=self._labels, predictions=self._predictions)
dumping_root = os.path.join(self.get_temp_dir(), 'tfdbg_dump_dir')
dumping_hook = hooks.DumpingDebugHook(dumping_root, log_usage=False)
try:
# Run the evaluation and verify the results:
accuracy_value = evaluation.evaluate_once(
'', checkpoint_path, log_dir, eval_op=update_op, final_op=value_op,
hooks=[dumping_hook])
self.assertAlmostEqual(accuracy_value, self._expected_accuracy)
dump = debug_data.DebugDumpDir(
glob.glob(os.path.join(dumping_root, 'run_*'))[0])
# Here we simply assert that the dumped data has been loaded and is
# non-empty. We do not care about the detailed model-internal tensors or
# their values.
self.assertTrue(dump.dumped_tensor_data)
finally:
if os.path.isdir(dumping_root):
shutil.rmtree(dumping_root)
def create_eval_metrics(self, noise):
predictions = np.array([0.1, 0.2, 0.3, 0.6 + noise])
labels = np.array([0.1, 0.2, 0.3, 0.6])
metrics = {
'accuracy': metrics_lib.accuracy(labels, predictions),
'auc': metrics_lib.auc(labels, predictions)
}
return metrics
def _accuracy_at_threshold(labels, predictions, weights, threshold, name=None):
with ops.name_scope(
name, 'accuracy_at_%s' % threshold,
(predictions, labels, weights, threshold)) as scope:
threshold_predictions = math_ops.to_float(
math_ops.greater_equal(predictions, threshold))
return metrics_lib.accuracy(
labels=labels, predictions=threshold_predictions, weights=weights,
name=scope)
def testRestoredModelPerformance(self):
checkpoint_path = os.path.join(self.get_temp_dir(), 'model.ckpt')
log_dir = os.path.join(self.get_temp_dir(), 'log_dir1/')
# First, save out the current model to a checkpoint:
self._prepareCheckpoint(checkpoint_path)
# Next, determine the metric to evaluate:
value_op, update_op = metrics.accuracy(
labels=self._labels, predictions=self._predictions)
# Run the evaluation and verify the results:
accuracy_value = evaluation.evaluate_once(
'', checkpoint_path, log_dir, eval_op=update_op, final_op=value_op)
self.assertAlmostEqual(accuracy_value, self._expected_accuracy)
def testFinalOpsOnEvaluationLoop(self):
value_op, update_op = metrics.accuracy(
labels=self._labels, predictions=self._predictions)
init_op = control_flow_ops.group(variables.global_variables_initializer(),
variables.local_variables_initializer())
# Create checkpoint and log directories:
chkpt_dir = os.path.join(self.get_temp_dir(), 'tmp_logs/')
gfile.MakeDirs(chkpt_dir)
logdir = os.path.join(self.get_temp_dir(), 'tmp_logs2/')
gfile.MakeDirs(logdir)
# Save initialized variables to a checkpoint directory:
saver = saver_lib.Saver()
with self.test_session() as sess:
init_op.run()
saver.save(sess, os.path.join(chkpt_dir, 'chkpt'))
class Object(object):
def __init__(self):
self.hook_was_run = False
obj = Object()
# Create a custom session run hook.
class CustomHook(session_run_hook.SessionRunHook):
def __init__(self, obj):
self.obj = obj
def end(self, session):
self.obj.hook_was_run = True
# Now, run the evaluation loop:
accuracy_value = evaluation.evaluation_loop(
'',
chkpt_dir,
logdir,
eval_op=update_op,
final_op=value_op,
hooks=[CustomHook(obj)],
max_number_of_evaluations=1)
self.assertAlmostEqual(accuracy_value, self._expected_accuracy)
# Validate that custom hook ran.
self.assertTrue(obj.hook_was_run)
def _eval_metric_ops(self, labels, probabilities, logits,
class_ids, weights, unweighted_loss):
"""Returns the Eval metric ops."""
with ops.name_scope(
None, 'metrics',
(labels, probabilities, logits, class_ids, weights, unweighted_loss)):
keys = metric_keys.MetricKeys
metric_ops = {
# Estimator already adds a metric for loss.
# TODO(xiejw): Any other metrics?
keys.LOSS_MEAN: metrics_lib.mean(
unweighted_loss, weights=weights, name=keys.LOSS_MEAN),
keys.ACCURACY: metrics_lib.accuracy(
labels=labels, predictions=class_ids, weights=weights,
name=keys.ACCURACY),
}
return metric_ops
def model_fn(self, mode, features, labels, params):
c = variable_scope.get_variable('c',
initializer=constant_op.constant(
10, dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = {'probabilities': math_ops.multiply(features, c)}
loss = losses.absolute_difference(
labels=labels,
predictions=predictions['probabilities'],
reduction=losses.Reduction.SUM)
metrics = {
'accuracy': metrics_lib.accuracy(labels,
predictions['probabilities']),
'auc': metrics_lib.auc(labels, predictions['probabilities'])
}
tensor_string_repr = str(features)
classes = constant_op.constant(re.search('(split_inputs/split:[0-9])',
tensor_string_repr).group(1),
dtype=dtypes.string)
export_outputs = {
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.PredictOutput(predictions),
'classification_output':
export_output.ClassificationOutput(predictions['probabilities'],
classes),
'classification_scores':
export_output.ClassificationOutput(
scores=predictions['probabilities']),
'classification_classes':
export_output.ClassificationOutput(classes=classes),
'regression_output':
export_output.RegressionOutput(predictions['probabilities']),
}
return model_fn_lib.EstimatorSpec(
mode=mode,
loss=math_ops.reduce_sum(loss),
eval_metric_ops=metrics,
predictions=predictions,
train_op=loss, # This train_op isn't actually used.
export_outputs=export_outputs)
def _eval_metric_ops(self, labels, class_ids, weights, unweighted_loss):
"""Returns the Eval metric ops."""
with ops.name_scope(None, 'metrics',
(labels, class_ids, weights, unweighted_loss)):
keys = metric_keys.MetricKeys
metric_ops = {
# Estimator already adds a metric for loss.
# TODO(xiejw): Any other metrics?
_summary_key(self._name, keys.LOSS_MEAN):
metrics_lib.mean(unweighted_loss,
weights=weights,
name=keys.LOSS_MEAN),
_summary_key(self._name, keys.ACCURACY):
metrics_lib.accuracy(labels=labels,
predictions=class_ids,
weights=weights,
name=keys.ACCURACY),
}
return metric_ops
def _eval_metric_ops(
self, labels, logits, logistic, scores, classes, unweighted_loss,
weights=None):
with ops.name_scope(
None, 'metrics',
(labels, logits, logistic, scores, classes, unweighted_loss, weights)):
keys = metric_keys.MetricKeys
labels_mean = _indicator_labels_mean(
labels=labels, weights=weights, name=keys.LABEL_MEAN)
metric_ops = {
# Estimator already adds a metric for loss.
keys.LOSS_MEAN: metrics_lib.mean(
unweighted_loss, weights=weights, name=keys.LOSS_MEAN),
keys.ACCURACY: metrics_lib.accuracy(
labels=labels, predictions=classes, weights=weights,
name=keys.ACCURACY),
keys.PREDICTION_MEAN: _predictions_mean(
predictions=logistic, weights=weights, name=keys.PREDICTION_MEAN),
keys.LABEL_MEAN: labels_mean,
keys.ACCURACY_BASELINE: _accuracy_baseline(labels_mean),
keys.AUC: _auc(
labels=labels, predictions=logistic, weights=weights,
name=keys.AUC),
keys.AUC_PR: _auc(
labels=labels, predictions=logistic, weights=weights, curve='PR',
name=keys.AUC_PR)
}
for threshold in self._thresholds:
accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold
metric_ops[accuracy_key] = _accuracy_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=accuracy_key)
# Precision for positive examples.
precision_key = keys.PRECISION_AT_THRESHOLD % threshold
metric_ops[precision_key] = _precision_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=precision_key)
# Recall for positive examples.
recall_key = keys.RECALL_AT_THRESHOLD % threshold
metric_ops[recall_key] = _recall_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=recall_key)
return metric_ops
def model_fn(self, mode, features, labels, params):
c = variable_scope.get_variable(
'c',
initializer=constant_op.constant(10, dtype=dtypes.float64),
dtype=dtypes.float64)
predictions = {'probabilities': math_ops.multiply(features, c)}
loss = losses.absolute_difference(
labels=labels,
predictions=predictions['probabilities'],
reduction=losses.Reduction.SUM)
metrics = {
'accuracy': metrics_lib.accuracy(labels, predictions['probabilities']),
'auc': metrics_lib.auc(labels, predictions['probabilities'])
}
tensor_string_repr = str(features)
classes = constant_op.constant(
re.search('(split_inputs/split:[0-9])', tensor_string_repr).group(1),
dtype=dtypes.string)
export_outputs = {
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
export_output.PredictOutput(predictions),
'classification_output':
export_output.ClassificationOutput(predictions['probabilities'],
classes),
'classification_scores':
export_output.ClassificationOutput(
scores=predictions['probabilities']),
'classification_classes':
export_output.ClassificationOutput(classes=classes),
'regression_output':
export_output.RegressionOutput(predictions['probabilities']),
}
return model_fn_lib.EstimatorSpec(
mode=mode,
loss=math_ops.reduce_sum(loss),
eval_metric_ops=metrics,
predictions=predictions,
train_op=loss, # This train_op isn't actually used.
export_outputs=export_outputs)
def testFinalOpsOnEvaluationLoop(self):
value_op, update_op = metrics.accuracy(labels=self._labels,
predictions=self._predictions)
init_op = control_flow_ops.group(
variables.global_variables_initializer(),
variables.local_variables_initializer())
# Create checkpoint and log directories:
chkpt_dir = tempfile.mkdtemp('tmp_logs')
logdir = tempfile.mkdtemp('tmp_logs2')
# Save initialized variables to a checkpoint directory:
saver = saver_lib.Saver()
with self.cached_session() as sess:
init_op.run()
saver.save(sess, os.path.join(chkpt_dir, 'chkpt'))
class Object(object):
def __init__(self):
self.hook_was_run = False
obj = Object()
# Create a custom session run hook.
class CustomHook(session_run_hook.SessionRunHook):
def __init__(self, obj):
self.obj = obj
def end(self, session):
self.obj.hook_was_run = True
# Now, run the evaluation loop:
accuracy_value = evaluation.evaluation_loop(
'',
chkpt_dir,
logdir,
eval_op=update_op,
final_op=value_op,
hooks=[CustomHook(obj)],
max_number_of_evaluations=1)
self.assertAlmostEqual(accuracy_value, self._expected_accuracy)
# Validate that custom hook ran.
self.assertTrue(obj.hook_was_run)
def _eval_metric_ops(
self, labels, logits, logistic, scores, classes, unweighted_loss,
weights=None):
with ops.name_scope(
None, 'metrics',
(labels, logits, logistic, scores, classes, unweighted_loss, weights)):
keys = metric_keys.MetricKeys
labels_mean = _indicator_labels_mean(
labels=labels, weights=weights, name=keys.LABEL_MEAN)
metric_ops = {
# Estimator already adds a metric for loss.
keys.LOSS_MEAN: metrics_lib.mean(
unweighted_loss, weights=weights, name=keys.LOSS_MEAN),
keys.ACCURACY: metrics_lib.accuracy(
labels=labels, predictions=classes, weights=weights,
name=keys.ACCURACY),
keys.PREDICTION_MEAN: _predictions_mean(
predictions=logistic, weights=weights, name=keys.PREDICTION_MEAN),
keys.LABEL_MEAN: labels_mean,
keys.ACCURACY_BASELINE: _accuracy_baseline(labels_mean),
keys.AUC: _auc(
labels=labels, predictions=logistic, weights=weights,
name=keys.AUC),
keys.AUC_PR: _auc(
labels=labels, predictions=logistic, weights=weights, curve='PR',
name=keys.AUC_PR)
}
for threshold in self._thresholds:
accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold
metric_ops[accuracy_key] = _accuracy_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=accuracy_key)
# Precision for positive examples.
precision_key = keys.PRECISION_AT_THRESHOLD % threshold
metric_ops[precision_key] = _precision_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=precision_key)
# Recall for positive examples.
recall_key = keys.RECALL_AT_THRESHOLD % threshold
metric_ops[recall_key] = _recall_at_threshold(
labels=labels, predictions=logistic, weights=weights,
threshold=threshold, name=recall_key)
return metric_ops
def testEvaluateWithFiniteInputs(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'evaluate_with_finite_inputs')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run evaluation. Inputs are fed through input producer for one epoch.
all_inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
all_labels = constant_op.constant(self._labels, dtype=dtypes.float32)
single_input, single_label = training.slice_input_producer(
[all_inputs, all_labels], num_epochs=1)
inputs, labels = training.batch([single_input, single_label],
batch_size=6,
allow_smaller_final_batch=True)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(predictions=predictions,
labels=labels)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=update_op,
final_ops={
'accuracy': accuracy,
'eval_steps': evaluation._get_or_create_eval_step()
},
hooks=[
evaluation._StopAfterNEvalsHook(None),
])
self.assertTrue(final_ops_values['accuracy'] > .99)
# Runs evaluation for 4 iterations. First 2 evaluate full batch of 6 inputs
# each; the 3rd iter evaluates the remaining 4 inputs, and the last one
# triggers an error which stops evaluation.
self.assertEqual(final_ops_values['eval_steps'], 4)
def testEvaluationLoopTimeoutWithTimeoutFn(self):
checkpoint_dir = tempfile.mkdtemp(
'evaluation_loop_timeout_with_timeout_fn')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run
inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
labels = constant_op.constant(self._labels, dtype=dtypes.float32)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(
predictions=predictions, labels=labels)
timeout_fn_calls = [0]
def timeout_fn():
timeout_fn_calls[0] += 1
return timeout_fn_calls[0] > 3
final_values = evaluation.evaluate_repeatedly(
checkpoint_dir=checkpoint_dir,
eval_ops=update_op,
final_ops={'accuracy': accuracy},
hooks=[
evaluation.StopAfterNEvalsHook(1),
],
eval_interval_secs=1,
max_number_of_evaluations=2,
timeout=0.1,
timeout_fn=timeout_fn)
# We should have evaluated once.
self.assertGreater(final_values['accuracy'], .99)
# And called 4 times the timeout fn
self.assertEqual(4, timeout_fn_calls[0])
def testEvaluatePerfectModel(self):
checkpoint_dir = os.path.join(self.get_temp_dir(),
'evaluate_perfect_model_once')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run
inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
labels = constant_op.constant(self._labels, dtype=dtypes.float32)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(
predictions=predictions, labels=labels)
checkpoint_path = saver.latest_checkpoint(checkpoint_dir)
final_ops_values = evaluation._evaluate_once(
checkpoint_path=checkpoint_path,
eval_ops=update_op,
final_ops={'accuracy': accuracy},
hooks=[evaluation._StopAfterNEvalsHook(1),])
self.assertTrue(final_ops_values['accuracy'] > .99)
def model_fn(features, labels, mode, params):
is_training = mode == tf.estimator.ModeKeys.TRAIN
NUM_CLASSES = len(params['label_vocab'])
module = hub.Module(TFHUB_CACHE_DIR, trainable=is_training and params['train_module'], name=params['module_name'])
bottleneck_tensor = module(features['inputs'])
with tf.compat.v1.name_scope('final_retrain_ops'):
logits = tf.compat.v1.layers.dense(bottleneck_tensor, units=1, trainable=is_training)
def train_op_fn(loss):
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
return optimizer.minimize(loss, global_step=tf.compat.v1.train.get_global_step())
if NUM_CLASSES == 2:
head = tf.estimator.BinaryClassHead(label_vocabulary=params['label_vocab'])
else:
head = tf.estimator.MultiClassHead(n_classes=NUM_CLASSES, label_vocabulary=params['label_vocab'])
spec = head.create_estimator_spec(
features, mode, logits, labels, train_op_fn=train_op_fn
)
if mode == tf.estimator.ModeKeys.TRAIN:
tf.compat.v1.summary.scalar('accuracy', metrics_lib.accuracy(labels, spec.predictions['classes'])[1])
logging_hook = logger_hook({"loss": spec.loss,"accuracy":
tf.compat.v1.metrics.accuracy(labels, spec.predictions['classes'])[1],
"step" : tf.compat.v1.train.get_or_create_global_step(), "steps_epoch": steps_epoch, "mode":"train"}, every_n_iter=summary_interval)
spec = spec._replace(training_hooks = [logging_hook])
if mode == tf.estimator.ModeKeys.EVAL:
print("evaluating")
logging_hook = logger_hook({"loss": spec.loss, "accuracy":
spec.eval_metric_ops['accuracy'][0], "step" :
tf.compat.v1.train.get_or_create_global_step(), "steps_epoch": steps_epoch, "mode": "eval"}, every_n_iter=summary_interval)
spec = spec._replace(evaluation_hooks = [logging_hook])
return spec
def testEvaluatePerfectModel(self):
checkpoint_dir = tempfile.mkdtemp('evaluate_perfect_model_repeated')
# Train a Model to completion:
self._train_model(checkpoint_dir, num_steps=300)
# Run
inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
labels = constant_op.constant(self._labels, dtype=dtypes.float32)
logits = logistic_classifier(inputs)
predictions = math_ops.round(logits)
accuracy, update_op = metrics.accuracy(
predictions=predictions, labels=labels)
final_values = evaluation.evaluate_repeatedly(
checkpoint_dir=checkpoint_dir,
eval_ops=update_op,
final_ops={'accuracy': accuracy},
hooks=[
evaluation.StopAfterNEvalsHook(1),
],
max_number_of_evaluations=1)
self.assertGreater(final_values['accuracy'], .99)
def _accuracy(predictions, targets, weights=None):
return metrics.accuracy(
labels=targets, predictions=predictions, weights=weights)
def _metric_fn(x):
labels = x["labels"]
predictions = x["predictions"]
return metrics.accuracy(labels, predictions)
def _eval_metric_ops(self, labels, logits, logistic, class_ids, weights,
weighted_sum_loss, example_weight_sum):
with ops.name_scope(None, 'metrics',
(labels, logits, logistic, class_ids, weights,
weighted_sum_loss, example_weight_sum)):
keys = metric_keys.MetricKeys
labels_mean = _indicator_labels_mean(
labels=labels, weights=weights, name=keys.LABEL_MEAN)
metric_ops = {
# Estimator already adds a metric for loss.
_summary_key(self._name, keys.LOSS_MEAN):
metrics_lib.mean(
# Both values and weights here are reduced, scalar Tensors.
# values is the actual mean we want -- weights represents the
# total weight of the batch and is needed to calculate
# update_op over many batches.
values=(weighted_sum_loss / example_weight_sum),
weights=example_weight_sum,
name=keys.LOSS_MEAN),
_summary_key(self._name, keys.ACCURACY):
metrics_lib.accuracy(
labels=labels,
predictions=class_ids,
weights=weights,
name=keys.ACCURACY),
_summary_key(self._name, keys.PREDICTION_MEAN):
_predictions_mean(
predictions=logistic,
weights=weights,
name=keys.PREDICTION_MEAN),
_summary_key(self._name, keys.LABEL_MEAN):
labels_mean,
_summary_key(self._name, keys.ACCURACY_BASELINE):
_accuracy_baseline(labels_mean),
_summary_key(self._name, keys.AUC):
_auc(
labels=labels,
predictions=logistic,
weights=weights,
name=keys.AUC),
_summary_key(self._name, keys.AUC_PR):
_auc(
labels=labels,
predictions=logistic,
weights=weights,
curve='PR',
name=keys.AUC_PR)
}
for threshold in self._thresholds:
accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
accuracy_key)] = _accuracy_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=accuracy_key)
# Precision for positive examples.
precision_key = keys.PRECISION_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
precision_key)] = _precision_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=precision_key)
# Recall for positive examples.
recall_key = keys.RECALL_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
recall_key)] = _recall_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=recall_key)
return metric_ops
def _accuracy(predictions, targets, weights=None):
return metrics.accuracy(labels=targets,
predictions=predictions,
weights=weights)
def _metric_fn(x): labels = x["labels"] predictions = x["predictions"] return metrics.accuracy(labels, predictions)
def _eval_metric_ops(self, labels, logits, logistic, class_ids, weights,
weighted_sum_loss, example_weight_sum):
with ops.name_scope(None, 'metrics',
(labels, logits, logistic, class_ids, weights,
weighted_sum_loss, example_weight_sum)):
keys = metric_keys.MetricKeys
labels_mean = _indicator_labels_mean(
labels=labels, weights=weights, name=keys.LABEL_MEAN)
metric_ops = {
# Estimator already adds a metric for loss.
_summary_key(self._name, keys.LOSS_MEAN):
metrics_lib.mean(
# Both values and weights here are reduced, scalar Tensors.
# values is the actual mean we want -- weights represents the
# total weight of the batch and is needed to calculate
# update_op over many batches.
values=(weighted_sum_loss / example_weight_sum),
weights=example_weight_sum,
name=keys.LOSS_MEAN),
_summary_key(self._name, keys.ACCURACY):
metrics_lib.accuracy(
labels=labels,
predictions=class_ids,
weights=weights,
name=keys.ACCURACY),
_summary_key(self._name, keys.PREDICTION_MEAN):
_predictions_mean(
predictions=logistic,
weights=weights,
name=keys.PREDICTION_MEAN),
_summary_key(self._name, keys.LABEL_MEAN):
labels_mean,
_summary_key(self._name, keys.ACCURACY_BASELINE):
_accuracy_baseline(labels_mean),
_summary_key(self._name, keys.AUC):
_auc(
labels=labels,
predictions=logistic,
weights=weights,
name=keys.AUC),
_summary_key(self._name, keys.AUC_PR):
_auc(
labels=labels,
predictions=logistic,
weights=weights,
curve='PR',
name=keys.AUC_PR)
}
for threshold in self._thresholds:
accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
accuracy_key)] = _accuracy_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=accuracy_key)
# Precision for positive examples.
precision_key = keys.PRECISION_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
precision_key)] = _precision_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=precision_key)
# Recall for positive examples.
recall_key = keys.RECALL_AT_THRESHOLD % threshold
metric_ops[_summary_key(self._name,
recall_key)] = _recall_at_threshold(
labels=labels,
predictions=logistic,
weights=weights,
threshold=threshold,
name=recall_key)
return metric_ops
