def data_stat(query): questions = geoserver_interface.download_questions(query) syntax_parses = questions_to_syntax_parses(questions, parser=False) annotations = geoserver_interface.download_semantics(query) unary_rules = [] binary_rules = [] semantic_trees = [] for pk, local_syntax_parses in syntax_parses.iteritems(): print pk for number, syntax_parse in local_syntax_parses.iteritems(): local_semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()] semantic_trees.extend(local_semantic_trees) print local_semantic_trees for semantic_tree in local_semantic_trees: unary_rules.extend(semantic_tree.get_unary_rules()) binary_rules.extend(semantic_tree.get_binary_rules()) tag_model = train_tag_model(syntax_parses, annotations) print "sentences: %d" % sum(len(question.sentence_words) for _, question in questions.iteritems()) print "words: %d" % (sum(len(words) for _, question in questions.iteritems() for _, words in question.sentence_words.iteritems())) print "literals: %d" % len(semantic_trees) print "unary rules: %d" % len(unary_rules) print "binary rules: %d" % len(binary_rules) print "" print "LEXICON" for key, s in tag_model.lexicon.iteritems(): print "%s: %s" % ("_".join(key), ", ".join(" ".join(ss) for ss in s))
def data_stat(query): questions = geoserver_interface.download_questions(query) syntax_parses = questions_to_syntax_parses(questions, parser=False) annotations = geoserver_interface.download_semantics(query) unary_rules = [] binary_rules = [] semantic_trees = [] for pk, local_syntax_parses in syntax_parses.iteritems(): print pk for number, syntax_parse in local_syntax_parses.iteritems(): local_semantic_trees = [ annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values() ] semantic_trees.extend(local_semantic_trees) print local_semantic_trees for semantic_tree in local_semantic_trees: unary_rules.extend(semantic_tree.get_unary_rules()) binary_rules.extend(semantic_tree.get_binary_rules()) tag_model = train_tag_model(syntax_parses, annotations) print "sentences: %d" % sum( len(question.sentence_words) for _, question in questions.iteritems()) print "words: %d" % (sum( len(words) for _, question in questions.iteritems() for _, words in question.sentence_words.iteritems())) print "literals: %d" % len(semantic_trees) print "unary rules: %d" % len(unary_rules) print "binary rules: %d" % len(binary_rules) print "" print "LEXICON" for key, s in tag_model.lexicon.iteritems(): print "%s: %s" % ("_".join(key), ", ".join(" ".join(ss) for ss in s))
def evaluate_opt_model(combined_model, syntax_parses, annotations, match_parses, thresholds): tps, fps, tns, fns = defaultdict(int), defaultdict(int), defaultdict(int), defaultdict(int) for pk, local_syntax_parses in syntax_parses.iteritems(): print "="*80 match_parse = match_parses[pk] for number, syntax_parse in local_syntax_parses.iteritems(): print pk, number opt_model = TextGreedyOptModel(combined_model) # opt_model = FullGreedyOptModel(combined_model, match_parse) pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()) pos_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC') tag_rules = combined_model.generate_tag_rules(syntax_parse) # tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees])) unary_rules = combined_model.generate_unary_rules(tag_rules) tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9) binary_rules = combined_model.generate_binary_rules(tag_rules) semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules) semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is')) semantic_trees = set(t for t in semantic_trees if combined_model.get_tree_score(t) > 0.01) neg_semantic_trees = semantic_trees - pos_semantic_trees for pst in pos_semantic_trees: print "pos:", combined_model.get_tree_score(pst), pst for nst in neg_semantic_trees: score = combined_model.get_tree_score(nst) if score > 0: print "neg:", score, nst print "" for th in thresholds: selected_trees = opt_model.optimize(semantic_trees, th) tp = len(selected_trees - neg_semantic_trees) fp = len(selected_trees - pos_semantic_trees) tn = len(neg_semantic_trees - selected_trees) fn = len(pos_semantic_trees - selected_trees) tps[th] += tp fps[th] += fp tns[th] += tn fns[th] += fn print "-"*80 prs = {} for th in thresholds: p = float(tps[th])/max(1,tps[th]+fps[th]) r = float(tps[th])/max(1,tps[th]+fns[th]) prs[th] = p, r return prs
def evaluate_opt_model(combined_model, syntax_parses, annotations, match_parses, thresholds): tps, fps, tns, fns = defaultdict(int), defaultdict(int), defaultdict(int), defaultdict(int) for pk, local_syntax_parses in syntax_parses.iteritems(): print "="*80 match_parse = match_parses[pk] for number, syntax_parse in local_syntax_parses.iteritems(): print pk, number opt_model = TextGreedyOptModel(combined_model) # opt_model = FullGreedyOptModel(combined_model, match_parse) pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()) pos_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC') tag_rules = combined_model.generate_tag_rules(syntax_parse) # tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees])) unary_rules = combined_model.generate_unary_rules(tag_rules) tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9) binary_rules = combined_model.generate_binary_rules(tag_rules) semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules) semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is')) semantic_trees = set(t for t in semantic_trees if combined_model.get_tree_score(t) > 0.01) neg_semantic_trees = semantic_trees - pos_semantic_trees for pst in pos_semantic_trees: print "pos:", combined_model.get_tree_score(pst), pst for nst in neg_semantic_trees: score = combined_model.get_tree_score(nst) if score > 0: print "neg:", score, nst print "" for th in thresholds: selected_trees = opt_model.optimize(semantic_trees, th) tp = len(selected_trees - neg_semantic_trees) fp = len(selected_trees - pos_semantic_trees) tn = len(neg_semantic_trees - selected_trees) fn = len(pos_semantic_trees - selected_trees) tps[th] += tp fps[th] += fp tns[th] += tn fns[th] += fn print "-"*80 prs = {} for th in thresholds: p = float(tps[th])/max(1,tps[th]+fps[th]) r = float(tps[th])/max(1,tps[th]+fns[th]) prs[th] = p, r return prs
def _annotated_unit_test(query): questions = geoserver_interface.download_questions(query) all_annotations = geoserver_interface.download_semantics(query) pk, question = questions.items()[0] choice_formulas = get_choice_formulas(question) label_data = geoserver_interface.download_labels(pk)[pk] diagram = open_image(question.diagram_path) graph_parse = diagram_to_graph_parse(diagram) core_parse = graph_parse.core_parse # core_parse.display_points() # core_parse.primitive_parse.display_primitives() match_parse = parse_match_from_known_labels(graph_parse, label_data) match_formulas = parse_match_formulas(match_parse) diagram_formulas = parse_confident_formulas(graph_parse) all_formulas = match_formulas + diagram_formulas for number, sentence_words in question.sentence_words.iteritems(): syntax_parse = stanford_parser.get_best_syntax_parse(sentence_words) annotation_nodes = [annotation_to_semantic_tree(syntax_parse, annotation) for annotation in all_annotations[pk][number].values()] expr_formulas = {key: prefix_to_formula(expression_parser.parse_prefix(expression)) for key, expression in question.sentence_expressions[number].iteritems()} truth_expr_formulas, value_expr_formulas = _separate_expr_formulas(expr_formulas) text_formula_parse = semantic_trees_to_text_formula_parse(annotation_nodes) completed_formulas = complete_formulas(text_formula_parse) grounded_formulas = [ground_formula(match_parse, formula, value_expr_formulas) for formula in completed_formulas+truth_expr_formulas] text_formulas = filter_formulas(flatten_formulas(grounded_formulas)) all_formulas.extend(text_formulas) reduced_formulas = reduce_formulas(all_formulas) for reduced_formula in reduced_formulas: score = evaluate(reduced_formula, core_parse.variable_assignment) scores = [evaluate(child, core_parse.variable_assignment) for child in reduced_formula.children] print reduced_formula, score, scores # core_parse.display_points() ans = solve(reduced_formulas, choice_formulas, assignment=core_parse.variable_assignment) print "ans:", ans if choice_formulas is None: attempted = True if abs(ans - float(question.answer)) < 0.01: correct = True else: correct = False else: attempted = True c = max(ans.iteritems(), key=lambda pair: pair[1].conf)[0] if c == int(question.answer): correct = True else: correct = False result = SimpleResult(query, False, attempted, correct) return result
def _annotated_unit_test(query): questions = geoserver_interface.download_questions(query) all_annotations = geoserver_interface.download_semantics(query) pk, question = questions.items()[0] choice_formulas = get_choice_formulas(question) label_data = geoserver_interface.download_labels(pk)[pk] diagram = open_image(question.diagram_path) graph_parse = diagram_to_graph_parse(diagram) core_parse = graph_parse.core_parse # core_parse.display_points() # core_parse.primitive_parse.display_primitives() match_parse = parse_match_from_known_labels(graph_parse, label_data) match_formulas = parse_match_formulas(match_parse) diagram_formulas = parse_confident_formulas(graph_parse) all_formulas = match_formulas + diagram_formulas for number, sentence_words in question.sentence_words.iteritems(): syntax_parse = stanford_parser.get_best_syntax_parse(sentence_words) annotation_nodes = [annotation_to_semantic_tree(syntax_parse, annotation) for annotation in all_annotations[pk][number].values()] expr_formulas = {key: prefix_to_formula(expression_parser.parse_prefix(expression)) for key, expression in question.sentence_expressions[number].iteritems()} truth_expr_formulas, value_expr_formulas = _separate_expr_formulas(expr_formulas) text_formula_parse = semantic_trees_to_text_formula_parse(annotation_nodes) completed_formulas = complete_formulas(text_formula_parse) grounded_formulas = [ground_formula(match_parse, formula, value_expr_formulas) for formula in completed_formulas+truth_expr_formulas] text_formulas = filter_formulas(flatten_formulas(grounded_formulas)) all_formulas.extend(text_formulas) reduced_formulas = reduce_formulas(all_formulas) for reduced_formula in reduced_formulas: score = evaluate(reduced_formula, core_parse.variable_assignment) scores = [evaluate(child, core_parse.variable_assignment) for child in reduced_formula.children] print reduced_formula, score, scores # core_parse.display_points() ans = solve(reduced_formulas, choice_formulas, assignment=core_parse.variable_assignment) print "ans:", ans if choice_formulas is None: attempted = True if abs(ans - float(question.answer)) < 0.01: correct = True else: correct = False else: attempted = True c = max(ans.iteritems(), key=lambda pair: pair[1].conf)[0] if c == int(question.answer): correct = True else: correct = False result = SimpleResult(query, False, attempted, correct) return result
def train_tag_model(syntax_parses, annotations): tm = NaiveTagModel() for pk, local_syntax_parses in syntax_parses.iteritems(): for number, syntax_parse in local_syntax_parses.iteritems(): semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()] assert isinstance(syntax_parse, SyntaxParse) local_tag_rules = set(itertools.chain(*[semantic_tree.get_tag_rules() for semantic_tree in semantic_trees])) tm.update(local_tag_rules) return tm
def train_semantic_model(tm, syntax_parses, annotations): um = RFUnaryModel() corem = RFCoreModel() ism = RFIsModel() ccm = NaiveCCModel(3) for pk, local_syntax_parses in syntax_parses.iteritems(): print "training:", pk for number, syntax_parse in local_syntax_parses.iteritems(): assert isinstance(syntax_parse, SyntaxParse) semantic_trees = [ annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values() ] # local_tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in semantic_trees])) local_tag_rules = tm.generate_tag_rules(syntax_parse) local_unary_rules = set( itertools.chain(*[ semantic_tree.get_unary_rules() for semantic_tree in semantic_trees ])) local_binary_rules = set( itertools.chain(*[ semantic_tree.get_binary_rules() for semantic_tree in semantic_trees ])) core_rules, is_rules, cc_rules = split_binary_rules( local_binary_rules) # Sanity check for ur in local_unary_rules: assert um.val_func(ur.parent_tag_rule, ur.child_tag_rule) for br in core_rules: assert corem.val_func(br.parent_tag_rule, br.child_a_tag_rule, br.child_b_tag_rule) um.update(local_tag_rules, local_unary_rules) corem.update(local_tag_rules, core_rules) ism.update(local_tag_rules, is_rules) #ccm.update(local_tag_rules, cc_rules) um.fit() corem.fit() ism.fit() #ccm.fit() cm = CombinedModel(tm, um, corem, ism, ccm) return cm
def train_semantic_model(tm, syntax_parses, annotations): um = RFUnaryModel() corem = RFCoreModel() ism = RFIsModel() ccm = NaiveCCModel(3) for pk, local_syntax_parses in syntax_parses.iteritems(): print "training:", pk for number, syntax_parse in local_syntax_parses.iteritems(): assert isinstance(syntax_parse, SyntaxParse) semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()] # local_tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in semantic_trees])) local_tag_rules = tm.generate_tag_rules(syntax_parse) local_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in semantic_trees])) local_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in semantic_trees])) core_rules, is_rules, cc_rules = split_binary_rules(local_binary_rules) # Sanity check for ur in local_unary_rules: assert um.val_func(ur.parent_tag_rule, ur.child_tag_rule) for br in core_rules: assert corem.val_func(br.parent_tag_rule, br.child_a_tag_rule, br.child_b_tag_rule) um.update(local_tag_rules, local_unary_rules) corem.update(local_tag_rules, core_rules) ism.update(local_tag_rules, is_rules) #ccm.update(local_tag_rules, cc_rules) um.fit() corem.fit() ism.fit() #ccm.fit() cm = CombinedModel(tm, um, corem, ism, ccm) return cm
def evaluate_rule_model(combined_model, syntax_parses, annotations, thresholds): all_pos_unary_rules = [] all_pos_core_rules = [] all_pos_is_rules = [] all_pos_cc_rules = [] all_neg_unary_rules = [] all_neg_core_rules = [] all_neg_is_rules = [] all_neg_cc_rules = [] all_pos_bool_semantic_trees = [] all_neg_bool_semantic_trees = [] for pk, local_syntax_parses in syntax_parses.iteritems(): print "\n\n\n" print pk for number, syntax_parse in local_syntax_parses.iteritems(): pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()) pos_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in pos_semantic_trees])) pos_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in pos_semantic_trees])) tag_rules = combined_model.tag_model.generate_tag_rules(syntax_parse) # tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees])) unary_rules = combined_model.generate_unary_rules(tag_rules) tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9) binary_rules = combined_model.generate_binary_rules(tag_rules) core_rules = combined_model.core_model.generate_binary_rules(tag_rules) is_rules = combined_model.is_model.generate_binary_rules(tag_rules) cc_rules = combined_model.cc_model.generate_binary_rules(tag_rules) pos_core_rules, pos_is_rules, pos_cc_rules = split_binary_rules(pos_binary_rules) span_pos_cc_rules = set(r.to_span_rule() for r in pos_cc_rules) negative_unary_rules = unary_rules - pos_unary_rules neg_core_rules = core_rules - pos_core_rules neg_is_rules = is_rules - pos_is_rules # neg_cc_rules = cc_rules - pos_cc_rules neg_cc_rules = set() pos_cc_rules = set() for r in cc_rules: if r.to_span_rule() in span_pos_cc_rules: pos_cc_rules.add(r) else: neg_cc_rules.add(r) all_pos_unary_rules.extend(pos_unary_rules) all_pos_core_rules.extend(pos_core_rules) all_pos_is_rules.extend(pos_is_rules) all_pos_cc_rules.extend(pos_cc_rules) all_neg_unary_rules.extend(negative_unary_rules) all_neg_core_rules.extend(neg_core_rules) all_neg_is_rules.extend(neg_is_rules) all_neg_cc_rules.extend(neg_cc_rules) pos_bool_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC') semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules) bool_semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is')) neg_bool_semantic_trees = bool_semantic_trees - pos_bool_semantic_trees all_pos_bool_semantic_trees.extend(pos_bool_semantic_trees) all_neg_bool_semantic_trees.extend(neg_bool_semantic_trees) for pst in pos_bool_semantic_trees: print "pos:", combined_model.get_tree_score(pst), pst print "" for nst in neg_bool_semantic_trees: score = combined_model.get_tree_score(nst) if score > 0: print "neg:", combined_model.get_tree_score(nst), nst unary_prs = combined_model.unary_model.get_prs(all_pos_unary_rules, all_neg_unary_rules, thresholds) core_prs = combined_model.core_model.get_prs(all_pos_core_rules, all_neg_core_rules, thresholds) is_prs = combined_model.is_model.get_prs(all_pos_is_rules, all_neg_is_rules, thresholds) cc_prs = combined_model.cc_model.get_prs(all_pos_cc_rules, all_neg_cc_rules, thresholds) core_tree_prs = combined_model.get_tree_prs(all_pos_bool_semantic_trees, all_neg_bool_semantic_trees, thresholds) return unary_prs, core_prs, is_prs, cc_prs, core_tree_prs
def evaluate_rule_model(combined_model, syntax_parses, annotations, thresholds): all_pos_unary_rules = [] all_pos_core_rules = [] all_pos_is_rules = [] all_pos_cc_rules = [] all_neg_unary_rules = [] all_neg_core_rules = [] all_neg_is_rules = [] all_neg_cc_rules = [] all_pos_bool_semantic_trees = [] all_neg_bool_semantic_trees = [] for pk, local_syntax_parses in syntax_parses.iteritems(): print "\n\n\n" print pk for number, syntax_parse in local_syntax_parses.iteritems(): pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation) for annotation in annotations[pk][number].values()) pos_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in pos_semantic_trees])) pos_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in pos_semantic_trees])) tag_rules = combined_model.tag_model.generate_tag_rules(syntax_parse) # tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees])) unary_rules = combined_model.generate_unary_rules(tag_rules) tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9) binary_rules = combined_model.generate_binary_rules(tag_rules) core_rules = combined_model.core_model.generate_binary_rules(tag_rules) is_rules = combined_model.is_model.generate_binary_rules(tag_rules) cc_rules = combined_model.cc_model.generate_binary_rules(tag_rules) pos_core_rules, pos_is_rules, pos_cc_rules = split_binary_rules(pos_binary_rules) span_pos_cc_rules = set(r.to_span_rule() for r in pos_cc_rules) negative_unary_rules = unary_rules - pos_unary_rules neg_core_rules = core_rules - pos_core_rules neg_is_rules = is_rules - pos_is_rules # neg_cc_rules = cc_rules - pos_cc_rules neg_cc_rules = set() pos_cc_rules = set() for r in cc_rules: if r.to_span_rule() in span_pos_cc_rules: pos_cc_rules.add(r) else: neg_cc_rules.add(r) all_pos_unary_rules.extend(pos_unary_rules) all_pos_core_rules.extend(pos_core_rules) all_pos_is_rules.extend(pos_is_rules) all_pos_cc_rules.extend(pos_cc_rules) all_neg_unary_rules.extend(negative_unary_rules) all_neg_core_rules.extend(neg_core_rules) all_neg_is_rules.extend(neg_is_rules) all_neg_cc_rules.extend(neg_cc_rules) pos_bool_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC') semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules) bool_semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is')) neg_bool_semantic_trees = bool_semantic_trees - pos_bool_semantic_trees all_pos_bool_semantic_trees.extend(pos_bool_semantic_trees) all_neg_bool_semantic_trees.extend(neg_bool_semantic_trees) for pst in pos_bool_semantic_trees: print "pos:", combined_model.get_tree_score(pst), pst print "" for nst in neg_bool_semantic_trees: score = combined_model.get_tree_score(nst) if score > 0: print "neg:", combined_model.get_tree_score(nst), nst unary_prs = combined_model.unary_model.get_prs(all_pos_unary_rules, all_neg_unary_rules, thresholds) core_prs = combined_model.core_model.get_prs(all_pos_core_rules, all_neg_core_rules, thresholds) is_prs = combined_model.is_model.get_prs(all_pos_is_rules, all_neg_is_rules, thresholds) cc_prs = combined_model.cc_model.get_prs(all_pos_cc_rules, all_neg_cc_rules, thresholds) core_tree_prs = combined_model.get_tree_prs(all_pos_bool_semantic_trees, all_neg_bool_semantic_trees, thresholds) return unary_prs, core_prs, is_prs, cc_prs, core_tree_prs