def test_feature_function(self): from experiment import evaluate_model from metrics import denotation_match_metrics from scoring import Model from geo880 import geo880_train_examples rules = (self.rules_optionals + self.rules_collection_entity + self.rules_types + self.rules_relations + self.rules_intersection + self.rules_superlatives + self.rules_reverse_joins) grammar = Unit3Grammar(rules=rules, annotators=self.annotators) def empty_denotation_feature(parse): features = defaultdict(float) if parse.denotation == (): features['empty_denotation'] += 1.0 return features weights = {'empty_denotation': -1.0} model = Model(grammar=grammar, feature_fn=empty_denotation_feature, weights=weights, executor=self.geobase.executor().execute) metric_values = evaluate_model(model=model, examples=geo880_train_examples, metrics=denotation_match_metrics(), print_examples=False) self.assertEqual(235, metric_values['denotation accuracy'])
def train_on_dev_experiment(): from metrics import denotation_match_metrics domain = ArithmeticDomain() train_test(model=domain.model(), train_examples=arithmetic_dev_examples, test_examples=domain.test_examples(), metrics=denotation_match_metrics(), training_metric=DenotationAccuracyMetric(), seed=1, print_examples=False)
def test_evaluate_model(self): from experiment import evaluate_model from metrics import denotation_match_metrics from scoring import Model from geo880 import geo880_train_examples rules = (self.rules_optionals + self.rules_collection_entity + self.rules_types + self.rules_relations + self.rules_intersection + self.rules_superlatives + self.rules_reverse_joins) grammar = Unit3Grammar(rules=rules, annotators=self.annotators) model = Model(grammar=grammar, executor=self.geobase.executor().execute) # Set print_examples=True and look for 'what state has the shortest # river?' and evaluate_model(model=model, examples=geo880_train_examples[:10], metrics=denotation_match_metrics(), print_examples=False)
def metrics(self): return denotation_match_metrics()
def metrics(self): """ The metrics that we are using for our experiments. """ return denotation_match_metrics()