def demo_learning_from_semantics(domain):
from example import Example
print('\nDemo of learning from semantics')
# Strip denotations, just to prove that we're learning from semantics.
def strip_denotation(example):
return Example(input=example.input, semantics=example.semantics)
examples = [
strip_denotation(example) for example in domain.train_examples()
]
model = domain.model()
model = latent_sgd(model=model,
examples=examples,
training_metric=SemanticsAccuracyMetric())
def test_learning_from_semantics(self):
"""
First 13 examples are used for training.
Last 4 examples are used for testing.
b_trn: performance metrics on training set before training
b_tst: performance metrics on test set before training
a_trn: performance metrics on training set after training
a_tst: performance metrics on test set after training
semantics accuracy: # of examples where parse at position 0 was correct.
denotation accuracy: # of examples where denotation of parse at position
0 was correct
"""
arithmetic_grammar = Grammar(self.arithmetic_rules)
arithmetic_examples = self.two_parse_examples + self.one_parse_examples
from executor import Executor
arithmetic_model = Model(
grammar=arithmetic_grammar,
feature_fn=Parse.operator_precedence_features,
weights=defaultdict(float), # Initialize with all weights at zero
executor=Executor.execute)
# Train based on correct/incorrect semantics
from metrics import SemanticsAccuracyMetric
b_trn, b_tst, a_trn, a_tst = arithmetic_model.train_test(
train_examples=arithmetic_examples[:13],
test_examples=arithmetic_examples[13:],
training_metric=SemanticsAccuracyMetric(),
seed=1)
# BEFORE SGD
self.assertEqual(b_trn['semantics accuracy'], 10)
self.assertEqual(b_trn['denotation accuracy'], 11)
self.assertEqual(b_tst['semantics accuracy'], 4)
self.assertEqual(b_tst['denotation accuracy'], 4)
# AFTER SGD
self.assertEqual(a_trn['semantics accuracy'], 13) # Improvement
self.assertEqual(a_trn['denotation accuracy'], 13) # Improvement
self.assertEqual(a_tst['semantics accuracy'], 4)
self.assertEqual(a_tst['denotation accuracy'], 4)
def train_test(model=None,
train_examples=[],
test_examples=[],
metrics=standard_metrics(),
training_metric=SemanticsAccuracyMetric(),
seed=None,
print_examples=False):
# print_grammar(model.grammar)
# print
print('%d training examples, %d test examples' %
(len(train_examples), len(test_examples)))
# 'Before' test
model.weights = defaultdict(float) # no weights
evaluate_model(model=model,
examples=train_examples,
examples_label='train',
metrics=metrics,
print_examples=print_examples)
evaluate_model(model=model,
examples=test_examples,
examples_label='test',
metrics=metrics,
print_examples=print_examples)
# Train
model = latent_sgd(model,
train_examples,
training_metric=training_metric,
seed=seed)
# 'After' test
evaluate_model(model=model,
examples=train_examples,
examples_label='train',
metrics=metrics,
print_examples=print_examples)
evaluate_model(model=model,
examples=test_examples,
examples_label='test',
metrics=metrics,
print_examples=print_examples)
def train_test(self,
train_examples=[],
test_examples=[],
metrics=standard_metrics(),
training_metric=SemanticsAccuracyMetric(),
seed=None,
print_examples=False):
# 'Before' test
self.weights = defaultdict(float) # no weights
before_train_metrics = self.evaluate(
examples=train_examples,
examples_label='train',
metrics=metrics,
print_examples=print_examples)
before_test_metrics = self.evaluate(
examples=test_examples,
examples_label='test',
metrics=metrics,
print_examples=print_examples)
# Train
model = self.train(train_examples, training_metric=training_metric, seed=seed)
# 'After' test
after_train_metrics = model.evaluate(
examples=train_examples,
examples_label='train',
metrics=metrics,
print_examples=print_examples)
after_test_metrics = model.evaluate(
examples=test_examples,
examples_label='test',
metrics=metrics,
print_examples=print_examples)
return before_train_metrics, before_test_metrics, after_train_metrics, after_test_metrics
def training_metric(self):
"""
Returns the evaluation metric which should be used to supervise training
for this domain.
"""
return SemanticsAccuracyMetric()