return DenotationAccuracyMetric() # GeobaseAnnotator ============================================================= # EXERCISE: Make it more robust, using string edit distance or minhashing. class GeobaseAnnotator(Annotator): def __init__(self, geobase): self.geobase = geobase def annotate(self, tokens): phrase = ' '.join(tokens) places = self.geobase.binaries_rev['name'][phrase] # places |= self.geobase.rev_index['abbreviation'][phrase] # TODO: $Entity? $Location? something that indicates type? return [('$Entity', place) for place in places] # demos and experiments ======================================================== if __name__ == '__main__': domain = GeoQueryDomain() evaluate_for_domain(domain, print_examples=False) # evaluate_dev_examples_for_domain(domain) # train_test_for_domain(domain, seed=1) # test_executor(domain) # sample_wins_and_losses(domain, metric=DenotationOracleAccuracyMetric()) # learn_lexical_semantics(domain, seed=1) # interact(domain, "the largest city in the largest state", T=0) # find_best_rules(domain)
Example(input='minus minus minus four', denotation=-4), Example(input='four plus two', denotation=6), Example(input='two minus three', denotation=-1), Example(input='minus four plus two', denotation=-2), ] def train_on_dev_experiment(): from metrics import denotation_match_metrics domain = ArithmeticDomain() train_test(model=domain.model(), train_examples=arithmetic_dev_examples, test_examples=domain.test_examples(), metrics=denotation_match_metrics(), training_metric=DenotationAccuracyMetric(), seed=1, print_examples=False) # ============================================================================== if __name__ == '__main__': evaluate_for_domain(ArithmeticDomain()) # train_test_for_domain(ArithmeticDomain(), seed=1, print_examples=False) # train_on_dev_experiment() # evaluate_for_domain(EagerArithmeticDomain()) # evaluate_dev_examples_for_domain(ArithmeticDomain()) # interact(ArithmeticDomain(), "two times two plus three") # learn_lexical_semantics(ArithmeticDomain(), seed=1) # generate(ArithmeticDomain().rules(), '$E')
Example(input='minus minus one', denotation=1), Example(input='minus minus minus four', denotation=-4), Example(input='four plus two', denotation=6), Example(input='two minus three', denotation=-1), Example(input='minus four plus two', denotation=-2), ] def train_on_dev_experiment(): from metrics import denotation_match_metrics domain = ArithmeticDomain() train_test(model=domain.model(), train_examples=arithmetic_dev_examples, test_examples=domain.test_examples(), metrics=denotation_match_metrics(), training_metric=DenotationAccuracyMetric(), seed=1, print_examples=False) # ============================================================================== if __name__ == '__main__': evaluate_for_domain(ArithmeticDomain()) # train_test_for_domain(ArithmeticDomain(), seed=1, print_examples=False) # train_on_dev_experiment() # evaluate_for_domain(EagerArithmeticDomain()) # evaluate_dev_examples_for_domain(ArithmeticDomain()) # interact(ArithmeticDomain(), "two times two plus three") # learn_lexical_semantics(ArithmeticDomain(), seed=1) # generate(ArithmeticDomain().rules(), '$E')
match = line_pattern.match(line) if not match: raise StandardError, 'unexpected line: %s' % line query = match.group(1) parses = parse_input(grammar, query) parses = filter(lambda parse: domain.is_travel_parse(parse), parses) if len(parses) > 0: print print query for parse in parses: print parse.semantics queries.append(' '.join(query)) if len(queries) % 1000 == 0: print print '-' * 80 print 'Processed %d queries' % len(queries) f.close() if __name__ == '__main__': domain = TravelDomain() evaluate_for_domain(domain, print_examples=False) # evaluate_dev_examples_for_domain(domain) # train_test_for_domain(domain, seed=1) # overtriggering_experiment() # interact(domain, 'directions from boston to austin by bike') # evaluate_for_domain(ContainsLocationDomain()) # filter_queries_containing_locations() # sample_wins_and_losses(domain=domain, metric=SemanticsOracleAccuracyMetric()) # learn_lexical_semantics(domain, seed=1)