def get_question_index(question):
ds = LC_Qaud_Linked("./data/LC-QUAD/linked_answer6.json")
ds.load()
ds.parse()
for i in range(0, len(ds.qapairs)):
if question == ds.qapairs[i].question.text:
print "Question id:", i
return i
test = json.load(f)
with open('data/LC-QUAD/linked_answer.json', 'r') as f:
data = json.load(f)
test_id=[]
for i in test:
test_id.append(i['id'])
test_data = []
for i in data:
if i['id'] in test_id:
test_data.append(i)
with open('data/LC-QUAD/linked_test.json', 'w') as f:
json.dump(test_data, f)
ds = LC_Qaud_Linked(path="./data/LC-QUAD/linked_test.json")
ds.load()
ds.parse()
if not ds.parser.kb.server_available:
logger.error("Server is not available. Please check the endpoint at: {}".format(ds.parser.kb.endpoint))
sys.exit(0)
output_file = 'lcquadtestanswer_output'
linker = Earl(path="data/LC-QUAD/entity_lcquad_test.json")
base_dir = "./output"
question_type_classifier_path = os.path.join(base_dir, "question_type_classifier")
double_relation_classifier_path = os.path.join(base_dir, "double_relation_classifier")
utility.makedirs(question_type_classifier_path)
utility.makedirs(double_relation_classifier_path)
# SELECT DISTINCT ?label
# WHERE {
# filter(lang(?label) = 'en').
# ns:m.03rjj rdfs:label ?label
# }
# ds = Qald(Qald.qald_8)
# ds.load()
# ds.parse()
# ds.print_pairs()
# ds = LC_Qaud()
# ds.load()
# ds.parse()
# ds.print_pairs()
ds = LC_Qaud_Linked()
ds.load()
ds.parse()
ds.print_pairs()
# for x in ds.qapairs[:10]:
# print x.sparql
# for y in x.sparql.uris:
# print y.type,
# print ""
target_uri = item.uris[0]
found = False
for e2_item in list2:
if target_uri in e2_item.uris:
found = True
break
if not found:
miss_match = True
break
return miss_match
if __name__ == "__main__":
stats = Stats()
ds = LC_Qaud_Linked("../data/LC-QUAD/linked.json")
ds.load()
ds.parse()
goldLinker = GoldLinker()
# earl = Earl("../data/LC-QUAD/EARL/output.json")
earl = Earl("../data/LC-QUAD/TagMeRelnliod/output_2300.json")
for qapair in tqdm(ds.qapairs):
e1, r1 = goldLinker.do(qapair)
e2, r2 = earl.do(qapair, force_gold=False, top=100)
if e2 is None and r2 is None:
continue
if len(e2) == 0:
stats.inc("earl_no_entity")
elif args.classifier == "naivebayes":
question_type_classifier = NaiveBayesClassifier(os.path.join(question_type_classifier_path, "naivebayes.model"))
double_relation_classifier = NaiveBayesClassifier(os.path.join(double_relation_classifier_path, "svm.model"))
stats = Stats()
t = args.dataset
output_file = args.file_name
if args.linker == 0:
linker = GoldLinker()
elif args.linker == 3:
linker = 3
else:
linker = Earl()
if t == 0:
ds = LC_Qaud_Linked(path=args.dataset_path)
ds.load()
ds.parse()
elif t == 1:
ds = WebQSP()
ds.load()
ds.parse()
ds.extend("./data/WebQuestionsSP/WebQSP.test.json")
elif t == 5:
ds = Qald(Qald.qald_5)
ds.load()
ds.parse()
elif t == 6:
ds = Qald(Qald.qald_6)
ds.load()
ds.parse()
for i in range(2):
if not all([
item.startswith('<') and item.endswith('>')
for item in items[i:i + 2]
]):
continue
key = ':'.join([items[i][1:-1], items[i + 1][1:-1]])
bloom.add(key)
with open(os.path.join(blooms_path, 'spo1.bloom'), 'wb') as f:
bloom.tofile(f)
with open(os.path.join(blooms_path, 'spo1.bloom'), 'rb') as f:
one_hop_bloom = BloomFilter.fromfile(f)
ds = LC_Qaud_Linked(
path=os.path.join(args.base_path, args.dataset_path))
ds.load()
ds.parse()
for row in ds.qapairs:
for item in row.sparql.where_clause:
if item[0].startswith('<'):
key = ':'.join([item[0][1:-1], item[1][1:-1]])
elif item[2].startswith('<'):
key = ':'.join([item[1][1:-1], item[2][1:-1]])
else:
key = ''
if '#type' not in key and key != '' and key not in one_hop_bloom:
print(key)
# print('http://dbpedia.org/resource/Actrius:http://dbpedia.org/property/director' in one_hop_bloom)
from parser.lc_quad_linked import LC_Qaud_Linked
import json
from kb.dbpedia import DBpedia
from parser.answerparser import AnswerParser
from common.utility.stats import Stats
from common.container.answerset import AnswerSet
from tqdm import tqdm
stats = Stats()
with open("../output/nliwod_origin.json") as data_file:
nliwod = json.load(data_file)
ds = LC_Qaud_Linked(path="../data/LC-QUAD/linked_answer6.json")
ds.load()
ds.parse()
kb = DBpedia()
parser = AnswerParser(kb)
i = 0
for qapair in tqdm(ds.qapairs):
nliwod_row = nliwod[i]
if qapair.id == nliwod_row["q_id"]:
query = nliwod_row["candidate"]
raw_result = kb.query(query)
result = AnswerSet(raw_result[1], parser.parse_queryresult)
nliwod_row["answer"] = raw_result[1]
if qapair.answerset == result:
stats.inc("correct")
nliwod_row["correct"] = True
else: