def test_learning_from_many_denotations(self):
"""
Large number of examples are used for training.
Last 4 arithmetic_examples are used for testing.
b_trn: performance metrics on training set before training
a_trn: performance metrics on training set after training
denotation accuracy: # of examples where denotation of parse at position
0 was correct
"""
arithmetic_grammar = Grammar(self.arithmetic_rules)
arithmetic_examples = self.two_parse_examples + self.one_parse_examples
from executor import Executor
arithmetic_model = Model(
grammar=arithmetic_grammar,
feature_fn=Parse.operator_precedence_features,
weights=defaultdict(float), # Initialize with all weights at zero
executor=Executor.execute)
from metrics import DenotationAccuracyMetric
from arithmetic import arithmetic_dev_examples
b_trn, b_tst, a_trn, a_tst = arithmetic_model.train_test(
train_examples=arithmetic_dev_examples,
test_examples=arithmetic_examples[13:],
training_metric=DenotationAccuracyMetric(),
seed=1)
# BEFORE SGD
self.assertEqual(b_trn['denotation accuracy'], 64)
# AFTER SGD
self.assertEqual(a_trn['denotation accuracy'], 92) # Improvement
def test_learning_from_denotation(self):
arithmetic_grammar = Grammar(self.arithmetic_rules)
arithmetic_examples = self.two_parse_examples + self.one_parse_examples
from executor import Executor
arithmetic_model = Model(
grammar=arithmetic_grammar,
feature_fn=Parse.operator_precedence_features,
weights=defaultdict(float), # Initialize with all weights at zero
executor=Executor.execute)
# Train based on correct/incorrect denotation
from metrics import DenotationAccuracyMetric
b_trn, b_tst, a_trn, a_tst = arithmetic_model.train_test(
train_examples=arithmetic_examples[:13],
test_examples=arithmetic_examples[13:],
training_metric=DenotationAccuracyMetric(),
seed=1)
# BEFORE SGD
self.assertEqual(b_trn['semantics accuracy'], 10)
self.assertEqual(b_tst['denotation accuracy'], 4)
# AFTER SGD
self.assertEqual(a_trn['semantics accuracy'], 12) # Improvement
self.assertEqual(a_trn['denotation accuracy'], 13) # Improvement
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
def demo_learning_from_denotations(domain):
from example import Example
print('\nDemo of learning from denotations')
# Strip semantics, just to prove that we're learning from denotations.
def strip_semantics(example):
return Example(input=example.input, denotation=example.denotation)
examples = [
strip_semantics(example) for example in domain.train_examples()
]
model = domain.model()
model = latent_sgd(model=model,
examples=examples,
training_metric=DenotationAccuracyMetric())
def training_metric(self):
return DenotationAccuracyMetric()
def training_metric(self):
"""
Similar to metrics.
"""
return DenotationAccuracyMetric()