def data_stat(query):
questions = geoserver_interface.download_questions(query)
syntax_parses = questions_to_syntax_parses(questions, parser=False)
annotations = geoserver_interface.download_semantics(query)
unary_rules = []
binary_rules = []
semantic_trees = []
for pk, local_syntax_parses in syntax_parses.iteritems():
print pk
for number, syntax_parse in local_syntax_parses.iteritems():
local_semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values()]
semantic_trees.extend(local_semantic_trees)
print local_semantic_trees
for semantic_tree in local_semantic_trees:
unary_rules.extend(semantic_tree.get_unary_rules())
binary_rules.extend(semantic_tree.get_binary_rules())
tag_model = train_tag_model(syntax_parses, annotations)
print "sentences: %d" % sum(len(question.sentence_words) for _, question in questions.iteritems())
print "words: %d" % (sum(len(words) for _, question in questions.iteritems() for _, words in question.sentence_words.iteritems()))
print "literals: %d" % len(semantic_trees)
print "unary rules: %d" % len(unary_rules)
print "binary rules: %d" % len(binary_rules)
print ""
print "LEXICON"
for key, s in tag_model.lexicon.iteritems():
print "%s: %s" % ("_".join(key), ", ".join(" ".join(ss) for ss in s))
def data_stat(query):
questions = geoserver_interface.download_questions(query)
syntax_parses = questions_to_syntax_parses(questions, parser=False)
annotations = geoserver_interface.download_semantics(query)
unary_rules = []
binary_rules = []
semantic_trees = []
for pk, local_syntax_parses in syntax_parses.iteritems():
print pk
for number, syntax_parse in local_syntax_parses.iteritems():
local_semantic_trees = [
annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values()
]
semantic_trees.extend(local_semantic_trees)
print local_semantic_trees
for semantic_tree in local_semantic_trees:
unary_rules.extend(semantic_tree.get_unary_rules())
binary_rules.extend(semantic_tree.get_binary_rules())
tag_model = train_tag_model(syntax_parses, annotations)
print "sentences: %d" % sum(
len(question.sentence_words) for _, question in questions.iteritems())
print "words: %d" % (sum(
len(words) for _, question in questions.iteritems()
for _, words in question.sentence_words.iteritems()))
print "literals: %d" % len(semantic_trees)
print "unary rules: %d" % len(unary_rules)
print "binary rules: %d" % len(binary_rules)
print ""
print "LEXICON"
for key, s in tag_model.lexicon.iteritems():
print "%s: %s" % ("_".join(key), ", ".join(" ".join(ss) for ss in s))
def evaluate_opt_model(combined_model, syntax_parses, annotations, match_parses, thresholds):
tps, fps, tns, fns = defaultdict(int), defaultdict(int), defaultdict(int), defaultdict(int)
for pk, local_syntax_parses in syntax_parses.iteritems():
print "="*80
match_parse = match_parses[pk]
for number, syntax_parse in local_syntax_parses.iteritems():
print pk, number
opt_model = TextGreedyOptModel(combined_model)
# opt_model = FullGreedyOptModel(combined_model, match_parse)
pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values())
pos_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC')
tag_rules = combined_model.generate_tag_rules(syntax_parse)
# tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees]))
unary_rules = combined_model.generate_unary_rules(tag_rules)
tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9)
binary_rules = combined_model.generate_binary_rules(tag_rules)
semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules)
semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is'))
semantic_trees = set(t for t in semantic_trees if combined_model.get_tree_score(t) > 0.01)
neg_semantic_trees = semantic_trees - pos_semantic_trees
for pst in pos_semantic_trees:
print "pos:", combined_model.get_tree_score(pst), pst
for nst in neg_semantic_trees:
score = combined_model.get_tree_score(nst)
if score > 0:
print "neg:", score, nst
print ""
for th in thresholds:
selected_trees = opt_model.optimize(semantic_trees, th)
tp = len(selected_trees - neg_semantic_trees)
fp = len(selected_trees - pos_semantic_trees)
tn = len(neg_semantic_trees - selected_trees)
fn = len(pos_semantic_trees - selected_trees)
tps[th] += tp
fps[th] += fp
tns[th] += tn
fns[th] += fn
print "-"*80
prs = {}
for th in thresholds:
p = float(tps[th])/max(1,tps[th]+fps[th])
r = float(tps[th])/max(1,tps[th]+fns[th])
prs[th] = p, r
return prs
def evaluate_opt_model(combined_model, syntax_parses, annotations, match_parses, thresholds):
tps, fps, tns, fns = defaultdict(int), defaultdict(int), defaultdict(int), defaultdict(int)
for pk, local_syntax_parses in syntax_parses.iteritems():
print "="*80
match_parse = match_parses[pk]
for number, syntax_parse in local_syntax_parses.iteritems():
print pk, number
opt_model = TextGreedyOptModel(combined_model)
# opt_model = FullGreedyOptModel(combined_model, match_parse)
pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values())
pos_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC')
tag_rules = combined_model.generate_tag_rules(syntax_parse)
# tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees]))
unary_rules = combined_model.generate_unary_rules(tag_rules)
tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9)
binary_rules = combined_model.generate_binary_rules(tag_rules)
semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules)
semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is'))
semantic_trees = set(t for t in semantic_trees if combined_model.get_tree_score(t) > 0.01)
neg_semantic_trees = semantic_trees - pos_semantic_trees
for pst in pos_semantic_trees:
print "pos:", combined_model.get_tree_score(pst), pst
for nst in neg_semantic_trees:
score = combined_model.get_tree_score(nst)
if score > 0:
print "neg:", score, nst
print ""
for th in thresholds:
selected_trees = opt_model.optimize(semantic_trees, th)
tp = len(selected_trees - neg_semantic_trees)
fp = len(selected_trees - pos_semantic_trees)
tn = len(neg_semantic_trees - selected_trees)
fn = len(pos_semantic_trees - selected_trees)
tps[th] += tp
fps[th] += fp
tns[th] += tn
fns[th] += fn
print "-"*80
prs = {}
for th in thresholds:
p = float(tps[th])/max(1,tps[th]+fps[th])
r = float(tps[th])/max(1,tps[th]+fns[th])
prs[th] = p, r
return prs
def _annotated_unit_test(query):
questions = geoserver_interface.download_questions(query)
all_annotations = geoserver_interface.download_semantics(query)
pk, question = questions.items()[0]
choice_formulas = get_choice_formulas(question)
label_data = geoserver_interface.download_labels(pk)[pk]
diagram = open_image(question.diagram_path)
graph_parse = diagram_to_graph_parse(diagram)
core_parse = graph_parse.core_parse
# core_parse.display_points()
# core_parse.primitive_parse.display_primitives()
match_parse = parse_match_from_known_labels(graph_parse, label_data)
match_formulas = parse_match_formulas(match_parse)
diagram_formulas = parse_confident_formulas(graph_parse)
all_formulas = match_formulas + diagram_formulas
for number, sentence_words in question.sentence_words.iteritems():
syntax_parse = stanford_parser.get_best_syntax_parse(sentence_words)
annotation_nodes = [annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in all_annotations[pk][number].values()]
expr_formulas = {key: prefix_to_formula(expression_parser.parse_prefix(expression))
for key, expression in question.sentence_expressions[number].iteritems()}
truth_expr_formulas, value_expr_formulas = _separate_expr_formulas(expr_formulas)
text_formula_parse = semantic_trees_to_text_formula_parse(annotation_nodes)
completed_formulas = complete_formulas(text_formula_parse)
grounded_formulas = [ground_formula(match_parse, formula, value_expr_formulas)
for formula in completed_formulas+truth_expr_formulas]
text_formulas = filter_formulas(flatten_formulas(grounded_formulas))
all_formulas.extend(text_formulas)
reduced_formulas = reduce_formulas(all_formulas)
for reduced_formula in reduced_formulas:
score = evaluate(reduced_formula, core_parse.variable_assignment)
scores = [evaluate(child, core_parse.variable_assignment) for child in reduced_formula.children]
print reduced_formula, score, scores
# core_parse.display_points()
ans = solve(reduced_formulas, choice_formulas, assignment=core_parse.variable_assignment)
print "ans:", ans
if choice_formulas is None:
attempted = True
if abs(ans - float(question.answer)) < 0.01:
correct = True
else:
correct = False
else:
attempted = True
c = max(ans.iteritems(), key=lambda pair: pair[1].conf)[0]
if c == int(question.answer):
correct = True
else:
correct = False
result = SimpleResult(query, False, attempted, correct)
return result
def _annotated_unit_test(query):
questions = geoserver_interface.download_questions(query)
all_annotations = geoserver_interface.download_semantics(query)
pk, question = questions.items()[0]
choice_formulas = get_choice_formulas(question)
label_data = geoserver_interface.download_labels(pk)[pk]
diagram = open_image(question.diagram_path)
graph_parse = diagram_to_graph_parse(diagram)
core_parse = graph_parse.core_parse
# core_parse.display_points()
# core_parse.primitive_parse.display_primitives()
match_parse = parse_match_from_known_labels(graph_parse, label_data)
match_formulas = parse_match_formulas(match_parse)
diagram_formulas = parse_confident_formulas(graph_parse)
all_formulas = match_formulas + diagram_formulas
for number, sentence_words in question.sentence_words.iteritems():
syntax_parse = stanford_parser.get_best_syntax_parse(sentence_words)
annotation_nodes = [annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in all_annotations[pk][number].values()]
expr_formulas = {key: prefix_to_formula(expression_parser.parse_prefix(expression))
for key, expression in question.sentence_expressions[number].iteritems()}
truth_expr_formulas, value_expr_formulas = _separate_expr_formulas(expr_formulas)
text_formula_parse = semantic_trees_to_text_formula_parse(annotation_nodes)
completed_formulas = complete_formulas(text_formula_parse)
grounded_formulas = [ground_formula(match_parse, formula, value_expr_formulas)
for formula in completed_formulas+truth_expr_formulas]
text_formulas = filter_formulas(flatten_formulas(grounded_formulas))
all_formulas.extend(text_formulas)
reduced_formulas = reduce_formulas(all_formulas)
for reduced_formula in reduced_formulas:
score = evaluate(reduced_formula, core_parse.variable_assignment)
scores = [evaluate(child, core_parse.variable_assignment) for child in reduced_formula.children]
print reduced_formula, score, scores
# core_parse.display_points()
ans = solve(reduced_formulas, choice_formulas, assignment=core_parse.variable_assignment)
print "ans:", ans
if choice_formulas is None:
attempted = True
if abs(ans - float(question.answer)) < 0.01:
correct = True
else:
correct = False
else:
attempted = True
c = max(ans.iteritems(), key=lambda pair: pair[1].conf)[0]
if c == int(question.answer):
correct = True
else:
correct = False
result = SimpleResult(query, False, attempted, correct)
return result
def train_tag_model(syntax_parses, annotations):
tm = NaiveTagModel()
for pk, local_syntax_parses in syntax_parses.iteritems():
for number, syntax_parse in local_syntax_parses.iteritems():
semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values()]
assert isinstance(syntax_parse, SyntaxParse)
local_tag_rules = set(itertools.chain(*[semantic_tree.get_tag_rules() for semantic_tree in semantic_trees]))
tm.update(local_tag_rules)
return tm
def train_semantic_model(tm, syntax_parses, annotations):
um = RFUnaryModel()
corem = RFCoreModel()
ism = RFIsModel()
ccm = NaiveCCModel(3)
for pk, local_syntax_parses in syntax_parses.iteritems():
print "training:", pk
for number, syntax_parse in local_syntax_parses.iteritems():
assert isinstance(syntax_parse, SyntaxParse)
semantic_trees = [
annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values()
]
# local_tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in semantic_trees]))
local_tag_rules = tm.generate_tag_rules(syntax_parse)
local_unary_rules = set(
itertools.chain(*[
semantic_tree.get_unary_rules()
for semantic_tree in semantic_trees
]))
local_binary_rules = set(
itertools.chain(*[
semantic_tree.get_binary_rules()
for semantic_tree in semantic_trees
]))
core_rules, is_rules, cc_rules = split_binary_rules(
local_binary_rules)
# Sanity check
for ur in local_unary_rules:
assert um.val_func(ur.parent_tag_rule, ur.child_tag_rule)
for br in core_rules:
assert corem.val_func(br.parent_tag_rule, br.child_a_tag_rule,
br.child_b_tag_rule)
um.update(local_tag_rules, local_unary_rules)
corem.update(local_tag_rules, core_rules)
ism.update(local_tag_rules, is_rules)
#ccm.update(local_tag_rules, cc_rules)
um.fit()
corem.fit()
ism.fit()
#ccm.fit()
cm = CombinedModel(tm, um, corem, ism, ccm)
return cm
def train_semantic_model(tm, syntax_parses, annotations):
um = RFUnaryModel()
corem = RFCoreModel()
ism = RFIsModel()
ccm = NaiveCCModel(3)
for pk, local_syntax_parses in syntax_parses.iteritems():
print "training:", pk
for number, syntax_parse in local_syntax_parses.iteritems():
assert isinstance(syntax_parse, SyntaxParse)
semantic_trees = [annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values()]
# local_tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in semantic_trees]))
local_tag_rules = tm.generate_tag_rules(syntax_parse)
local_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in semantic_trees]))
local_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in semantic_trees]))
core_rules, is_rules, cc_rules = split_binary_rules(local_binary_rules)
# Sanity check
for ur in local_unary_rules:
assert um.val_func(ur.parent_tag_rule, ur.child_tag_rule)
for br in core_rules:
assert corem.val_func(br.parent_tag_rule, br.child_a_tag_rule, br.child_b_tag_rule)
um.update(local_tag_rules, local_unary_rules)
corem.update(local_tag_rules, core_rules)
ism.update(local_tag_rules, is_rules)
#ccm.update(local_tag_rules, cc_rules)
um.fit()
corem.fit()
ism.fit()
#ccm.fit()
cm = CombinedModel(tm, um, corem, ism, ccm)
return cm
def evaluate_rule_model(combined_model, syntax_parses, annotations, thresholds):
all_pos_unary_rules = []
all_pos_core_rules = []
all_pos_is_rules = []
all_pos_cc_rules = []
all_neg_unary_rules = []
all_neg_core_rules = []
all_neg_is_rules = []
all_neg_cc_rules = []
all_pos_bool_semantic_trees = []
all_neg_bool_semantic_trees = []
for pk, local_syntax_parses in syntax_parses.iteritems():
print "\n\n\n"
print pk
for number, syntax_parse in local_syntax_parses.iteritems():
pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values())
pos_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in pos_semantic_trees]))
pos_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in pos_semantic_trees]))
tag_rules = combined_model.tag_model.generate_tag_rules(syntax_parse)
# tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees]))
unary_rules = combined_model.generate_unary_rules(tag_rules)
tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9)
binary_rules = combined_model.generate_binary_rules(tag_rules)
core_rules = combined_model.core_model.generate_binary_rules(tag_rules)
is_rules = combined_model.is_model.generate_binary_rules(tag_rules)
cc_rules = combined_model.cc_model.generate_binary_rules(tag_rules)
pos_core_rules, pos_is_rules, pos_cc_rules = split_binary_rules(pos_binary_rules)
span_pos_cc_rules = set(r.to_span_rule() for r in pos_cc_rules)
negative_unary_rules = unary_rules - pos_unary_rules
neg_core_rules = core_rules - pos_core_rules
neg_is_rules = is_rules - pos_is_rules
# neg_cc_rules = cc_rules - pos_cc_rules
neg_cc_rules = set()
pos_cc_rules = set()
for r in cc_rules:
if r.to_span_rule() in span_pos_cc_rules:
pos_cc_rules.add(r)
else:
neg_cc_rules.add(r)
all_pos_unary_rules.extend(pos_unary_rules)
all_pos_core_rules.extend(pos_core_rules)
all_pos_is_rules.extend(pos_is_rules)
all_pos_cc_rules.extend(pos_cc_rules)
all_neg_unary_rules.extend(negative_unary_rules)
all_neg_core_rules.extend(neg_core_rules)
all_neg_is_rules.extend(neg_is_rules)
all_neg_cc_rules.extend(neg_cc_rules)
pos_bool_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC')
semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules)
bool_semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is'))
neg_bool_semantic_trees = bool_semantic_trees - pos_bool_semantic_trees
all_pos_bool_semantic_trees.extend(pos_bool_semantic_trees)
all_neg_bool_semantic_trees.extend(neg_bool_semantic_trees)
for pst in pos_bool_semantic_trees:
print "pos:", combined_model.get_tree_score(pst), pst
print ""
for nst in neg_bool_semantic_trees:
score = combined_model.get_tree_score(nst)
if score > 0:
print "neg:", combined_model.get_tree_score(nst), nst
unary_prs = combined_model.unary_model.get_prs(all_pos_unary_rules, all_neg_unary_rules, thresholds)
core_prs = combined_model.core_model.get_prs(all_pos_core_rules, all_neg_core_rules, thresholds)
is_prs = combined_model.is_model.get_prs(all_pos_is_rules, all_neg_is_rules, thresholds)
cc_prs = combined_model.cc_model.get_prs(all_pos_cc_rules, all_neg_cc_rules, thresholds)
core_tree_prs = combined_model.get_tree_prs(all_pos_bool_semantic_trees, all_neg_bool_semantic_trees, thresholds)
return unary_prs, core_prs, is_prs, cc_prs, core_tree_prs
def evaluate_rule_model(combined_model, syntax_parses, annotations, thresholds):
all_pos_unary_rules = []
all_pos_core_rules = []
all_pos_is_rules = []
all_pos_cc_rules = []
all_neg_unary_rules = []
all_neg_core_rules = []
all_neg_is_rules = []
all_neg_cc_rules = []
all_pos_bool_semantic_trees = []
all_neg_bool_semantic_trees = []
for pk, local_syntax_parses in syntax_parses.iteritems():
print "\n\n\n"
print pk
for number, syntax_parse in local_syntax_parses.iteritems():
pos_semantic_trees = set(annotation_to_semantic_tree(syntax_parse, annotation)
for annotation in annotations[pk][number].values())
pos_unary_rules = set(itertools.chain(*[semantic_tree.get_unary_rules() for semantic_tree in pos_semantic_trees]))
pos_binary_rules = set(itertools.chain(*[semantic_tree.get_binary_rules() for semantic_tree in pos_semantic_trees]))
tag_rules = combined_model.tag_model.generate_tag_rules(syntax_parse)
# tag_rules = set(itertools.chain(*[t.get_tag_rules() for t in pos_semantic_trees]))
unary_rules = combined_model.generate_unary_rules(tag_rules)
tag_rules = filter_tag_rules(combined_model.unary_model, tag_rules, unary_rules, 0.9)
binary_rules = combined_model.generate_binary_rules(tag_rules)
core_rules = combined_model.core_model.generate_binary_rules(tag_rules)
is_rules = combined_model.is_model.generate_binary_rules(tag_rules)
cc_rules = combined_model.cc_model.generate_binary_rules(tag_rules)
pos_core_rules, pos_is_rules, pos_cc_rules = split_binary_rules(pos_binary_rules)
span_pos_cc_rules = set(r.to_span_rule() for r in pos_cc_rules)
negative_unary_rules = unary_rules - pos_unary_rules
neg_core_rules = core_rules - pos_core_rules
neg_is_rules = is_rules - pos_is_rules
# neg_cc_rules = cc_rules - pos_cc_rules
neg_cc_rules = set()
pos_cc_rules = set()
for r in cc_rules:
if r.to_span_rule() in span_pos_cc_rules:
pos_cc_rules.add(r)
else:
neg_cc_rules.add(r)
all_pos_unary_rules.extend(pos_unary_rules)
all_pos_core_rules.extend(pos_core_rules)
all_pos_is_rules.extend(pos_is_rules)
all_pos_cc_rules.extend(pos_cc_rules)
all_neg_unary_rules.extend(negative_unary_rules)
all_neg_core_rules.extend(neg_core_rules)
all_neg_is_rules.extend(neg_is_rules)
all_neg_cc_rules.extend(neg_cc_rules)
pos_bool_semantic_trees = set(t for t in pos_semantic_trees if t.content.signature.id != 'CC')
semantic_forest = SemanticForest(tag_rules, unary_rules, binary_rules)
bool_semantic_trees = semantic_forest.get_semantic_trees_by_type("truth").union(semantic_forest.get_semantic_trees_by_type('is'))
neg_bool_semantic_trees = bool_semantic_trees - pos_bool_semantic_trees
all_pos_bool_semantic_trees.extend(pos_bool_semantic_trees)
all_neg_bool_semantic_trees.extend(neg_bool_semantic_trees)
for pst in pos_bool_semantic_trees:
print "pos:", combined_model.get_tree_score(pst), pst
print ""
for nst in neg_bool_semantic_trees:
score = combined_model.get_tree_score(nst)
if score > 0:
print "neg:", combined_model.get_tree_score(nst), nst
unary_prs = combined_model.unary_model.get_prs(all_pos_unary_rules, all_neg_unary_rules, thresholds)
core_prs = combined_model.core_model.get_prs(all_pos_core_rules, all_neg_core_rules, thresholds)
is_prs = combined_model.is_model.get_prs(all_pos_is_rules, all_neg_is_rules, thresholds)
cc_prs = combined_model.cc_model.get_prs(all_pos_cc_rules, all_neg_cc_rules, thresholds)
core_tree_prs = combined_model.get_tree_prs(all_pos_bool_semantic_trees, all_neg_bool_semantic_trees, thresholds)
return unary_prs, core_prs, is_prs, cc_prs, core_tree_prs
