def test_evaluate_grammar_with_reverse_joins(self): from experiment import sample_wins_and_losses from geoquery import GeoQueryDomain from metrics import DenotationOracleAccuracyMetric from scoring import Model rules = (self.rules_optionals + self.rules_collection_entity + self.rules_types + self.rules_relations + self.rules_intersection + self.rules_superlatives + self.rules_reverse_joins) grammar = Unit3Grammar(rules=rules, annotators=self.annotators) model = Model(grammar=grammar, executor=self.geobase.executor().execute) metric = DenotationOracleAccuracyMetric() # If printing=True, prints a sampling of wins (correct semantics in # first parse) and losses on the dataset. metric_values = sample_wins_and_losses(domain=self.domain, model=model, metric=metric, seed=1, printing=False) self.assertEqual(11562, metric_values['number of parses']) self.assertEqual(152, metric_values['denotation accuracy'])
def test_training_data4(self): from experiment import sample_wins_and_losses from metrics import SemanticsOracleAccuracyMetric from scoring import Model from travel import TravelDomain from geonames import GeoNamesAnnotator domain = TravelDomain() rules = self.rules_travel + self.rules_travel_locations + self.rules_travel_modes + self.rules_travel_triggers + self.rules_request_types + self.rules_optionals grammar = Unit2Grammar(rules=rules, annotators=[GeoNamesAnnotator(live_requests=False)]) model = Model(grammar=grammar) metric = SemanticsOracleAccuracyMetric() # If printing=True, prints a sampling of wins (correct semantics in # first parse) and losses on the dataset. metric_values = sample_wins_and_losses(domain=domain, model=model, metric=metric, seed=31, printing=False)
def test_evaluate_simple_grammar(self): from experiment import sample_wins_and_losses from metrics import DenotationOracleAccuracyMetric from scoring import Model rules = self.rules_optionals + self.rules_collection_entity grammar = Unit2Grammar(rules=rules, annotators=self.annotators) model = Model(grammar=grammar, executor=self.geobase.executor().execute) metric = DenotationOracleAccuracyMetric() # If printing=True, prints a sampling of wins (correct semantics in # first parse) and losses on the dataset. metric_values = sample_wins_and_losses(domain=self.domain, model=model, metric=metric, seed=1, printing=False) self.assertEqual(17, metric_values['number of parses'])
Creates lexical rules for $Entity using the Geobase knowledge base """ def __init__(self, geobase): self.geobase = geobase def annotate(self, tokens): """ TODO: Create an annotator will annotate the $Entity category with the name :param tokens: List of tokens to make up query :return: list of tuples of the form ('$Entity', place) Hint: Note that the GraphKB datastructure allows for reverse queries by calling binaries_rev['name'][query] which will return a list of names for the given query. """ # raise NotImplementedError("GeobaseAnnotator.annotate") listoftuples = [] query = ' '.join(tokens) for key in self.geobase.binaries_rev.keys(): for value in self.geobase.binaries_rev[key][query]: listoftuples.append(('$Entity', value)) return listoftuples if __name__ == '__main__': domain = GeoQueryDomain() # Note that you can keep running this script to evaluate your rules. # Hint: First complete the annotator and the grammar class before running these tests for part 2. # time1 = time.time() sample_wins_and_losses(domain=domain) # print("\nTime Taken = ",(time.time() - time1))