def computed_every_grounded_graph_f1_graphq(input_file):
from datasets_interface.question_interface import graphquestion_interface
for structure_path in os.listdir(input_file):
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answers_mid_set = graphquestion_interface.get_answers_mid_by_question(
structure.question)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
new_system_answers_list = []
for system_answer in set(grounded_graph.denotation):
if isinstance(system_answer, int):
new_system_answers_list.append(str(system_answer))
else:
new_system_answers_list.append(system_answer)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answers_mid_set, new_system_answers_list)
grounded_graph.f1_score = f1
grounded_graph.recall_score = recall
grounded_graph.precision_score = precision
if f1 > 0:
print(
structure_path, f1
) # print(structure_path, gold_answers_mid_set, new_system_answers_list, f1)
structure.gold_answer = gold_answers_mid_set # update answers by answer mid list ["Kimberly-Clark"] ['en.kimberly-clark']
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_cwq(input_file):
from datasets_interface.question_interface import complexwebquestion_interface
all_structure_path = os.listdir(input_file)
error_list = []
for structure_path in all_structure_path:
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
try:
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answer_mid_set = complexwebquestion_interface.get_answers_by_question(
structure.question)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_mid_set, system_denotation_set)
grounded_graph.f1_score = f1
grounded_graph.recall_score = recall
grounded_graph.precision_score = precision
write_structure_file(structure_list, input_file + structure_path)
except Exception as e:
print('error')
error_list.append(structure_path)
print('error_list:\t', error_list)
def computed_every_grounded_graph_f1_webq_name(input_file, answer_file,
mid_to_names_file):
# from datasets_interface.freebase import webquestions_interface
# from evaluation.webq_denotation import webq_mid_to_names_process
#------------------------------------------------
#read qid-to-answers
qid_to_answers_dict = dict()
lines = read_list(answer_file)
for line in lines:
cols = line.split('\t')
qid_to_answers_dict[cols[0]] = eval(cols[2])
#------------------------------------------------
# mid to names dict
mid_to_names_dict = dict()
lines = read_list(mid_to_names_file)
for line in lines:
cols = line.split('\t')
mid = cols[1]
names = list(eval(cols[2]))
mid_to_names_dict[mid] = names
#------------------------------------------------
all_structure_path = os.listdir(input_file)
for structure_path in all_structure_path:
structure_with_grounded_graphq_file = input_file + structure_path
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
qid = structure.qid
gold_answer_names_set = evaluation_utils.search_for_answers_by_id(
qid, qid_to_answers_dict)
print(structure_path, '#gold:\t', gold_answer_names_set)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_names_set = set()
for denotation_mid in grounded_graph.denotation:
denotation_name = evaluation_utils.get_name_by_mid(
denotation_mid, mid_to_names_dict)
print('###denotation:\t', denotation_mid,
denotation_name)
if denotation_name is not None:
system_denotation_names_set.add(denotation_name)
else:
print(denotation_mid, '#####error!!!',
denotation_name)
print('#gold:\t', gold_answer_names_set, '#system:\t',
system_denotation_names_set)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_names_set, system_denotation_names_set)
if f1 > 0.0:
print('#result:\t', f1)
grounded_graph.f1_score = f1
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_graphq(input_file):
from grounding.grounding_args import test_qid_to_answers_mid_dict, train_qid_to_answers_mid_dict
for structure_path in os.listdir(input_file): #all_structure_path
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answers_mid_set = []
qid = structure.qid
if qid in test_qid_to_answers_mid_dict:
gold_answers_mid_set = test_qid_to_answers_mid_dict[qid]
elif qid in train_qid_to_answers_mid_dict:
gold_answers_mid_set = train_qid_to_answers_mid_dict[qid]
#[80] -> ['80']
new_gold_answers_set = set()
for gold_answer in gold_answers_mid_set:
if isinstance(gold_answer, int):
new_gold_answers_set.add(str(gold_answer))
else:
new_gold_answers_set.add(gold_answer)
gold_answers_mid_set = list(new_gold_answers_set)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
new_system_answers_set = set()
for system_answer in system_denotation_set:
if isinstance(system_answer, int):
new_system_answers_set.add(str(system_answer))
else:
new_system_answers_set.add(system_answer)
new_system_answers_set = list(new_system_answers_set)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answers_mid_set, new_system_answers_set)
print(structure_path, gold_answers_mid_set,
new_system_answers_set, f1)
grounded_graph.f1_score = f1
if f1 > 0:
print(f1)
# update answers by answer mid list ["Kimberly-Clark"] ['en.kimberly-clark']
structure.gold_answer = gold_answers_mid_set
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_webq_mid(input_file, answer_file):
#read qid-to-answers
all_structure_path = os.listdir(input_file)
for structure_path in all_structure_path:
structure_with_grounded_graphq_file = input_file + structure_path
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
qid = structure.qid
gold_answer_mid_set = evaluation_utils.search_for_answers_by_id(
qid, qid_to_answers_dict)
print(structure_path, gold_answer_mid_set)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_mid_set, system_denotation_set)
grounded_graph.f1_score = f1
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_cwq(input_file):
all_structure_path = os.listdir(input_file)
for structure_path in all_structure_path:
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answer_mid_set = evaluation_utils.get_gold_answers(
structure.gold_answer)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_mid_set, system_denotation_set)
grounded_graph.f1_score = f1
if f1 > 0:
print(f1)
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_lcquad(input_file):
from datasets_interface.question_interface import lcquad_1_0_interface
for structure_path in os.listdir(input_file):
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answer_mid_set = lcquad_1_0_interface.get_answers_by_question(
structure.question) #['http://dbpedia.org/resource/Colorado']
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_mid_set, system_denotation_set)
grounded_graph.f1_score = f1
grounded_graph.recall_score = recall
grounded_graph.precision_score = precision
write_structure_file(structure_list, input_file + structure_path)
def computed_every_grounded_graph_f1_lcquad(input_file):
from datasets_interface.question_interface import lcquad_1_0_interface
all_structure_path = os.listdir(input_file)
for structure_path in all_structure_path:
structure_with_grounded_graphq_file = input_file + structure_path
print(structure_path)
structure_list = read_structure_file(
structure_with_grounded_graphq_file)
for structure in structure_list:
gold_answer_mid_set = lcquad_1_0_interface.get_answers_by_question(
structure.question)
print('#gold answer:\t', gold_answer_mid_set)
for ungrounded_graph in structure.ungrounded_graph_forest:
for grounded_graph in ungrounded_graph.get_grounded_graph_forest(
):
system_denotation_set = set(grounded_graph.denotation)
recall, precision, f1 = sempre_evaluation.computeF1(
gold_answer_mid_set, system_denotation_set)
grounded_graph.f1_score = f1
if f1 > 0:
print(f1)
write_structure_file(structure_list, input_file + structure_path)