return DenotationAccuracyMetric()
# GeobaseAnnotator =============================================================
# EXERCISE: Make it more robust, using string edit distance or minhashing.
class GeobaseAnnotator(Annotator):
def __init__(self, geobase):
self.geobase = geobase
def annotate(self, tokens):
phrase = ' '.join(tokens)
places = self.geobase.binaries_rev['name'][phrase]
# places |= self.geobase.rev_index['abbreviation'][phrase]
# TODO: $Entity? $Location? something that indicates type?
return [('$Entity', place) for place in places]
# demos and experiments ========================================================
if __name__ == '__main__':
domain = GeoQueryDomain()
evaluate_for_domain(domain, print_examples=False)
# evaluate_dev_examples_for_domain(domain)
# train_test_for_domain(domain, seed=1)
# test_executor(domain)
# sample_wins_and_losses(domain, metric=DenotationOracleAccuracyMetric())
# learn_lexical_semantics(domain, seed=1)
# interact(domain, "the largest city in the largest state", T=0)
# find_best_rules(domain)
Example(input='minus minus minus four', denotation=-4),
Example(input='four plus two', denotation=6),
Example(input='two minus three', denotation=-1),
Example(input='minus four plus two', denotation=-2),
]
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
# ==============================================================================
if __name__ == '__main__':
evaluate_for_domain(ArithmeticDomain())
# train_test_for_domain(ArithmeticDomain(), seed=1, print_examples=False)
# train_on_dev_experiment()
# evaluate_for_domain(EagerArithmeticDomain())
# evaluate_dev_examples_for_domain(ArithmeticDomain())
# interact(ArithmeticDomain(), "two times two plus three")
# learn_lexical_semantics(ArithmeticDomain(), seed=1)
# generate(ArithmeticDomain().rules(), '$E')
Example(input='minus minus one', denotation=1),
Example(input='minus minus minus four', denotation=-4),
Example(input='four plus two', denotation=6),
Example(input='two minus three', denotation=-1),
Example(input='minus four plus two', denotation=-2),
]
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
# ==============================================================================
if __name__ == '__main__':
evaluate_for_domain(ArithmeticDomain())
# train_test_for_domain(ArithmeticDomain(), seed=1, print_examples=False)
# train_on_dev_experiment()
# evaluate_for_domain(EagerArithmeticDomain())
# evaluate_dev_examples_for_domain(ArithmeticDomain())
# interact(ArithmeticDomain(), "two times two plus three")
# learn_lexical_semantics(ArithmeticDomain(), seed=1)
# generate(ArithmeticDomain().rules(), '$E')
match = line_pattern.match(line)
if not match:
raise StandardError, 'unexpected line: %s' % line
query = match.group(1)
parses = parse_input(grammar, query)
parses = filter(lambda parse: domain.is_travel_parse(parse), parses)
if len(parses) > 0:
print
print query
for parse in parses:
print parse.semantics
queries.append(' '.join(query))
if len(queries) % 1000 == 0:
print
print '-' * 80
print 'Processed %d queries' % len(queries)
f.close()
if __name__ == '__main__':
domain = TravelDomain()
evaluate_for_domain(domain, print_examples=False)
# evaluate_dev_examples_for_domain(domain)
# train_test_for_domain(domain, seed=1)
# overtriggering_experiment()
# interact(domain, 'directions from boston to austin by bike')
# evaluate_for_domain(ContainsLocationDomain())
# filter_queries_containing_locations()
# sample_wins_and_losses(domain=domain, metric=SemanticsOracleAccuracyMetric())
# learn_lexical_semantics(domain, seed=1)
