def test_next_production_rule_accuracy_with_length(self):
partial_sequence_lengths = tf.placeholder(tf.int32, shape=[None])
value, update_op = metrics.next_production_rule_accuracy(
next_production_rules=self.next_production_rules,
predict_next_production_rules=self.predict_next_production_rules,
partial_sequence_lengths=partial_sequence_lengths,
target_length=1)
with self.test_session() as sess:
sess.run(tf.local_variables_initializer())
sess.run(
update_op,
feed_dict={
self.next_production_rules:
self.next_production_rules_values_0,
self.predict_next_production_rules:
self.predict_next_production_rules_values_0,
partial_sequence_lengths: [42, 42, 1, 42]})
self.assertAlmostEqual(value.eval(), 1.)
sess.run(
update_op,
feed_dict={
self.next_production_rules:
self.next_production_rules_values_1,
self.predict_next_production_rules:
self.predict_next_production_rules_values_1,
partial_sequence_lengths: [42, 42, 42, 42, 42, 42]})
self.assertAlmostEqual(value.eval(), 1.)
def test_next_production_rule_accuracy(self):
value, update_op = metrics.next_production_rule_accuracy(
next_production_rules=self.next_production_rules,
predict_next_production_rules=self.predict_next_production_rules)
with self.test_session() as sess:
sess.run(tf.local_variables_initializer())
sess.run(
update_op,
feed_dict={
self.next_production_rules:
self.next_production_rules_values_0,
self.predict_next_production_rules:
self.predict_next_production_rules_values_0})
self.assertAlmostEqual(value.eval(), 0.25)
sess.run(
update_op,
feed_dict={
self.next_production_rules:
self.next_production_rules_values_1,
self.predict_next_production_rules:
self.predict_next_production_rules_values_1})
self.assertAlmostEqual(value.eval(), 0.4)
def model_fn(features, labels, mode, params, grammar):
"""Builds the model graph.
Args:
features: Dict of tensors.
labels: Dict of tensors, or None if mode == INFER.
mode: tf.estimator.ModeKeys execution mode.
params: HParams object containing model hyperparameters.
grammar: arithmetic_grammar.Grammar object.
Returns:
A ModelFnOps object defining predictions, loss, and train_op.
"""
if mode != tf.estimator.ModeKeys.PREDICT:
tf.summary.text('expression_string',
features['expression_string'][:10])
tf.summary.text('production_rules',
tf.constant(grammar.grammar_to_string()))
# Make features easier to look up.
with tf.variable_scope('features'):
features = {
key: tf.identity(value, name=key)
for key, value in six.iteritems(features)
}
embedding_layer = networks.partial_sequence_encoder(
features=features,
symbolic_properties=core.hparams_list_value(
params.symbolic_properties),
numerical_points=core.hparams_list_value(params.numerical_points),
num_production_rules=grammar.num_production_rules,
embedding_size=params.embedding_size)
logits = networks.build_stacked_gru_model(
embedding_layer=embedding_layer,
partial_sequence_length=features['partial_sequence_length'],
gru_hidden_sizes=params.gru_hidden_sizes,
num_output_features=grammar.num_production_rules,
bidirectional=params.bidirectional)
predictions = {'logits': tf.identity(logits, name='predictions/logits')}
predictions.update({
name: tf.identity(tensor, name='predictions/%s' % name)
for name, tensor in six.iteritems(
mask_logits(logits, features['next_production_rule_mask']))
})
predictions['next_production_rule'] = tf.argmax(
predictions['masked_probabilities'],
axis=1,
name='predictions/next_production_rule')
if mode == tf.estimator.ModeKeys.PREDICT:
return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions)
# NOTE(leeley): The mask cannot be applied directly on logits. Because 0
# logit is still corresponding to a positive probability. Since
# tf.losses.sparse_softmax_cross_entropy() only works for logits rather than
# probabilities, I convert probabilities back to logits by tf.log(). Since
# the probabilities for grammarly invalid production rules are 0, to avoid
# numerical issue of log(0), I added a small number 1e-10.
loss = tf.losses.sparse_softmax_cross_entropy(
labels, tf.log(predictions['masked_probabilities'] + 1e-10))
# Configure the training op for TRAIN mode.
if mode == tf.estimator.ModeKeys.TRAIN:
train_op = contrib_layers.optimize_loss(
loss=loss,
global_step=tf.train.get_global_step(),
learning_rate=core.learning_rate_decay(
initial_learning_rate=params.learning_rate,
decay_steps=params.learning_rate_decay_steps,
decay_rate=params.learning_rate_decay_rate),
optimizer=params.optimizer,
summaries=contrib_layers.OPTIMIZER_SUMMARIES)
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
train_op=train_op)
# Add evaluation metrics for EVAL mode.
eval_metric_ops = {
'eval_loss':
tf.metrics.mean(loss),
'count':
contrib_metrics.count(labels),
'next_production_rule_valid_ratio':
metrics.next_production_rule_valid_ratio(
unmasked_probabilities_batch=predictions['unmasked_probabilities'],
next_production_rule_masks=features['next_production_rule_mask']),
'next_production_rule_accuracy':
metrics.next_production_rule_accuracy(
next_production_rules=labels,
predict_next_production_rules=predictions['next_production_rule']),
}
for target_length in range(1, params.max_length + 1):
eval_metric_ops[
'next_production_rule_info/length_%d' %
target_length] = metrics.next_production_rule_info_batch_text_summary(
expression_strings=features['expression_string'],
partial_sequences=features['partial_sequence'],
partial_sequence_lengths=features['partial_sequence_length'],
next_production_rules=labels,
unmasked_probabilities_batch=predictions[
'unmasked_probabilities'],
masked_probabilities_batch=predictions['masked_probabilities'],
grammar=grammar,
target_length=target_length)
eval_metric_ops[
'next_production_rule_valid_ratio/length_%d' %
target_length] = metrics.next_production_rule_valid_ratio(
unmasked_probabilities_batch=predictions[
'unmasked_probabilities'],
next_production_rule_masks=features[
'next_production_rule_mask'],
partial_sequence_lengths=features['partial_sequence_length'],
target_length=target_length)
eval_metric_ops[
'next_production_rule_accuracy/length_%d' %
target_length] = metrics.next_production_rule_accuracy(
next_production_rules=labels,
predict_next_production_rules=predictions[
'next_production_rule'],
partial_sequence_lengths=features['partial_sequence_length'],
target_length=target_length)
if params.num_expressions_per_condition > 0:
with tf.variable_scope('conditional_generation'):
match_ratio = tf.placeholder(tf.float32,
shape=[None],
name='match_ratio')
fail_ratio = tf.placeholder(tf.float32,
shape=[None],
name='fail_ratio')
eval_metric_ops.update({
'generation_match_ratio':
tf.metrics.mean(match_ratio),
'generation_fail_ratio':
tf.metrics.mean(fail_ratio),
})
return tf.estimator.EstimatorSpec(mode=mode,
loss=loss,
eval_metric_ops=eval_metric_ops)
