def test_feature_function(self):
from experiment import evaluate_model
from metrics import denotation_match_metrics
from scoring import Model
from geo880 import geo880_train_examples
rules = (self.rules_optionals + self.rules_collection_entity +
self.rules_types + self.rules_relations +
self.rules_intersection + self.rules_superlatives +
self.rules_reverse_joins)
grammar = Unit3Grammar(rules=rules, annotators=self.annotators)
def empty_denotation_feature(parse):
features = defaultdict(float)
if parse.denotation == ():
features['empty_denotation'] += 1.0
return features
weights = {'empty_denotation': -1.0}
model = Model(grammar=grammar,
feature_fn=empty_denotation_feature,
weights=weights,
executor=self.geobase.executor().execute)
metric_values = evaluate_model(model=model,
examples=geo880_train_examples,
metrics=denotation_match_metrics(),
print_examples=False)
self.assertEqual(235, metric_values['denotation accuracy'])
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
def test_evaluate_model(self):
from experiment import evaluate_model
from metrics import denotation_match_metrics
from scoring import Model
from geo880 import geo880_train_examples
rules = (self.rules_optionals + self.rules_collection_entity +
self.rules_types + self.rules_relations +
self.rules_intersection + self.rules_superlatives +
self.rules_reverse_joins)
grammar = Unit3Grammar(rules=rules, annotators=self.annotators)
model = Model(grammar=grammar,
executor=self.geobase.executor().execute)
# Set print_examples=True and look for 'what state has the shortest
# river?' and
evaluate_model(model=model,
examples=geo880_train_examples[:10],
metrics=denotation_match_metrics(),
print_examples=False)
def metrics(self):
return denotation_match_metrics()
def metrics(self):
"""
The metrics that we are using for our experiments.
"""
return denotation_match_metrics()
