def test_evaluate_grammar_with_reverse_joins(self):
from experiment import sample_wins_and_losses
from geoquery import GeoQueryDomain
from metrics import DenotationOracleAccuracyMetric
from scoring import Model
rules = (self.rules_optionals + self.rules_collection_entity +
self.rules_types + self.rules_relations +
self.rules_intersection + self.rules_superlatives +
self.rules_reverse_joins)
grammar = Unit3Grammar(rules=rules, annotators=self.annotators)
model = Model(grammar=grammar,
executor=self.geobase.executor().execute)
metric = DenotationOracleAccuracyMetric()
# If printing=True, prints a sampling of wins (correct semantics in
# first parse) and losses on the dataset.
metric_values = sample_wins_and_losses(domain=self.domain,
model=model,
metric=metric,
seed=1,
printing=False)
self.assertEqual(11562, metric_values['number of parses'])
self.assertEqual(152, metric_values['denotation accuracy'])
def test_training_data4(self):
from experiment import sample_wins_and_losses
from metrics import SemanticsOracleAccuracyMetric
from scoring import Model
from travel import TravelDomain
from geonames import GeoNamesAnnotator
domain = TravelDomain()
rules = self.rules_travel + self.rules_travel_locations + self.rules_travel_modes + self.rules_travel_triggers + self.rules_request_types + self.rules_optionals
grammar = Unit2Grammar(rules=rules, annotators=[GeoNamesAnnotator(live_requests=False)])
model = Model(grammar=grammar)
metric = SemanticsOracleAccuracyMetric()
# If printing=True, prints a sampling of wins (correct semantics in
# first parse) and losses on the dataset.
metric_values = sample_wins_and_losses(domain=domain, model=model, metric=metric, seed=31, printing=False)
def test_evaluate_simple_grammar(self):
from experiment import sample_wins_and_losses
from metrics import DenotationOracleAccuracyMetric
from scoring import Model
rules = self.rules_optionals + self.rules_collection_entity
grammar = Unit2Grammar(rules=rules, annotators=self.annotators)
model = Model(grammar=grammar,
executor=self.geobase.executor().execute)
metric = DenotationOracleAccuracyMetric()
# If printing=True, prints a sampling of wins (correct semantics in
# first parse) and losses on the dataset.
metric_values = sample_wins_and_losses(domain=self.domain,
model=model,
metric=metric,
seed=1,
printing=False)
self.assertEqual(17, metric_values['number of parses'])
Creates lexical rules for $Entity using the Geobase knowledge base
"""
def __init__(self, geobase):
self.geobase = geobase
def annotate(self, tokens):
"""
TODO: Create an annotator will annotate the $Entity category with the name
:param tokens: List of tokens to make up query
:return: list of tuples of the form ('$Entity', place)
Hint: Note that the GraphKB datastructure allows for reverse queries by calling binaries_rev['name'][query]
which will return a list of names for the given query.
"""
# raise NotImplementedError("GeobaseAnnotator.annotate")
listoftuples = []
query = ' '.join(tokens)
for key in self.geobase.binaries_rev.keys():
for value in self.geobase.binaries_rev[key][query]:
listoftuples.append(('$Entity', value))
return listoftuples
if __name__ == '__main__':
domain = GeoQueryDomain()
# Note that you can keep running this script to evaluate your rules.
# Hint: First complete the annotator and the grammar class before running these tests for part 2.
# time1 = time.time()
sample_wins_and_losses(domain=domain)
# print("\nTime Taken = ",(time.time() - time1))
