def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
def train_on_dev_experiment():
from metrics import denotation_match_metrics
domain = ArithmeticDomain()
train_test(model=domain.model(),
train_examples=arithmetic_dev_examples,
test_examples=domain.test_examples(),
metrics=denotation_match_metrics(),
training_metric=DenotationAccuracyMetric(),
seed=1,
print_examples=False)
def special_geo_evaluate(grammar=None, feature_fn=geo_domain.features):
# Build the model by hand so that we can see all the pieces:
geo_mod = Model(grammar=grammar,
feature_fn=feature_fn,
weights=geo_domain.weights(),
executor=geo_domain.execute)
# This can be done with less fuss using experiment.train_test_for_domain,
# but we want full access to the model, metrics, etc.
train_test(model=geo_mod,
train_examples=geo_domain.train_examples(),
test_examples=geo_domain.test_examples(),
metrics=geo_domain.metrics(),
training_metric=geo_domain.training_metric(),
seed=0,
print_examples=False)
def special_geo_evaluate(grammar=None, feature_fn=geo_domain.features):
# Build the model by hand so that we can see all the pieces:
geo_mod = Model(
grammar=grammar,
feature_fn=feature_fn,
weights=geo_domain.weights(),
executor=geo_domain.execute)
# This can be done with less fuss using experiment.train_test_for_domain,
# but we want full access to the model, metrics, etc.
train_test(
model=geo_mod,
train_examples=geo_domain.train_examples(),
test_examples=geo_domain.test_examples(),
metrics=geo_domain.metrics(),
training_metric=geo_domain.training_metric(),
seed=0,
print_examples=False)
travel_domain = TravelDomain()
travel_grammar = travel_domain.grammar()
def basic_feature_function(parse):
"""Features for the rule used for the root node and its children"""
features = defaultdict(float)
features[str(parse.rule)] += 1.0
for child in parse.children:
features[str(child.rule)] += 1.0
return features
# This code evaluates the current grammar:
train_test(
model=Model(grammar=travel_grammar, feature_fn=basic_feature_function),
train_examples=travel_train_examples,
test_examples=travel_test_examples,
print_examples=False)
# ### Question 4
#
# With the default travel grammar, many of the errors on training
# examples occur because the origin isn't marked by "from". You might
# have noticed that "directions New York to Philadelphia" is not
# handled properly in our opening example. Other examples include
# "transatlantic cruise southampton to tampa", "fly boston to myrtle
# beach spirit airlines", and "distance usa to peru". __Your tasks__:
# (i) extend the grammar with a single rule to handle examples like
# these, and run another evaluation using this expanded grammar
# (submit your completion of the following starter code); (ii) in
travel_grammar = travel_domain.grammar()
def basic_feature_function(parse):
"""Features for the rule used for the root node and its children"""
features = defaultdict(float)
features[str(parse.rule)] += 1.0
for child in parse.children:
features[str(child.rule)] += 1.0
return features
# This code evaluates the current grammar:
train_test(model=Model(grammar=travel_grammar,
feature_fn=basic_feature_function),
train_examples=travel_train_examples,
test_examples=travel_test_examples,
print_examples=False)
# ### Question 4
#
# With the default travel grammar, many of the errors on training examples occur because the origin
# isn't marked by "from". You might have noticed that "directions New York to Philadelphia"
# is not handled properly in our opening example. Other examples include
# "transatlantic cruise southampton to tampa",
# "fly boston to myrtle beach spirit airlines", and
# "distance usa to peru". __Your tasks__: (i) extend the grammar with a single rule to handle examples
# like these, and run another evaluation using this expanded grammar (submit your completion
# of the following starter code); (ii) in 1–2 sentences,
# summarize what happened to the post-training performance metrics when this rule was added.