class TupleExtractor(object):
def __init__(self):
try:
self.entity2relations_dic = pickle.load(
open('../data/entity2relation_dic.pkl', 'rb'))
except:
self.entity2relations_dic = {}
try:
self.sentencepair2sim = pickle.load(
open('../data/sentencepair2sim_dic.pkl', 'rb'))
except:
self.sentencepair2sim = {}
self.simmer = BertSim()
self.simmer.set_mode(tf.estimator.ModeKeys.PREDICT)
print('tuples extractor loaded')
def extract_tuples(self, candidate_entitys, question):
''''''
candidate_tuples = {}
for entity in candidate_entitys:
#得到该实体的所有关系路径
starttime = time.time()
relations = GetRelationPaths(entity)
mention = candidate_entitys[entity][0]
for r in relations:
this_tuple = tuple([entity] + r) #生成候选tuple
predicates = [relation[1:-1]
for relation in r] #python-list 关系名列表
human_question = '的'.join([mention] + predicates)
score = [entity] + [s for s in candidate_entitys[entity][0:1]
] #初始化特征
try:
sim2 = self.sentencepair2sim[question + human_question]
except:
sim2 = self.simmer.predict(question, human_question)[0][1]
self.sentencepair2sim[question + human_question] = sim2
self.sentencepair2sim[question + human_question] = sim2
score.append(sim2)
candidate_tuples[this_tuple] = score
print('====查询候选关系并计算特征耗费%.2f秒====' % (time.time() - starttime))
return candidate_tuples
def GetCandidateAns(self, corpus):
'''根据mention，得到所有候选实体,进一步去知识库检索候选答案
候选答案格式为tuple(entity,relation1,relation2) 这样便于和标准答案对比
'''
true_num = 0
hop2_num = 0
hop2_true_num = 0
all_tuples_num = 0
for i in range(len(corpus)):
dic = corpus[i]
question = dic['question']
gold_tuple = dic['gold_tuple']
gold_entitys = dic['gold_entitys']
candidate_entitys = dic['candidate_entity_filter']
candidate_tuples = self.extract_tuples(candidate_entitys, question)
print(i)
print(question)
all_tuples_num += len(candidate_tuples)
dic['candidate_tuples'] = candidate_tuples
#判断gold tuple是否包含在candidate_tuples_list中
if_true = 0
for thistuple in candidate_tuples:
if len(gold_tuple) == len(
set(gold_tuple).intersection(set(thistuple))):
if_true = 1
break
if if_true == 1:
true_num += 1
if len(gold_tuple) <= 3 and len(gold_entitys) == 1:
hop2_true_num += 1
if len(gold_tuple) <= 3 and len(gold_entitys) == 1:
hop2_num += 1
print('所有问题里，候选答案能覆盖标准查询路径的比例为:%.3f' % (true_num / len(corpus)))
print('单实体问题中，候选答案能覆盖标准查询路径的比例为:%.3f' % (hop2_true_num / hop2_num))
print('平均每个问题的候选答案数量为:%.3f' % (all_tuples_num / len(corpus)))
pickle.dump(self.entity2relations_dic,
open('../data/entity2relation_dic.pkl', 'wb'))
pickle.dump(self.sentencepair2sim,
open('../data/sentencepair2sim_dic.pkl', 'wb'))
return corpus
class AnswerCandidate(Candidate):
def __init__(self,
entity2relations_dict='data/entity2relations_dict.pkl',
seqPair2similarity_dict='data/seqPair2similarity_dict.pkl'):
self._entity2relations = self._load_dict(entity2relations_dict)
self._seqPair2similarity = self._load_dict(seqPair2similarity_dict)
self._similarity_dict_path = seqPair2similarity_dict
self._relation_paths_dict_path = entity2relations_dict
self._model = BertSim()
self._model.mode = tf.estimator.ModeKeys.PREDICT
def _similarity_of(self, faked, seq):
k = faked + seq
if k not in self._seqPair2similarity:
self._seqPair2similarity[k] = self._model.predict(faked, seq)
return self._seqPair2similarity[k]
def _relation_paths_of(self, entity):
if entity not in self._entity2relations:
return []
return self._entity2relations[entity]
def _candidates_of(self, entity2feats, question):
answer2feats = {}
for entity, feats in entity2feats.items():
relation_paths = self._relation_paths_of(entity)
if not relation_paths:
continue
mention = feats[0]
for relations in relation_paths:
answer = (entity, *relations)
predicates = [spo[1:-1] for spo in relations]
hypothesis = '的'.join([mention] + predicates)
feats = [
entity, mention,
self._similarity_of(hypothesis, question)
]
answer2feats[answer] = feats
return answer2feats
def candidates_of(self, subject2feats: Dict[str, list], question: str):
return self._candidates_of(subject2feats, question)
def add_candidates_to_corpus(self, corpus: Corpus):
num_answers = .0
num_2hop = .0
num_cover = {'all': .0, '2hop': .0}
for i, sample in enumerate(corpus):
question = sample['question']
gold_answer = sample['gold_tuple']
gold_entities = sample['gold_entitys']
subject_linked = sample['subject_linked']
candidate_answers = self._candidates_of(subject_linked, question)
num_answers += len(candidate_answers)
sample['candidate_answer'] = candidate_answers
ever_cover = False
for answer in candidate_answers:
if set(answer).issuperset(gold_answer):
ever_cover = True
print('* Question: ({}){}\n*\tAnswer: {}'.format(
i, question, answer))
break
if ever_cover:
num_cover['all'] += 1
if len(gold_answer) <= 3 and len(gold_entities) == 1:
num_cover['2hop'] += 1
if len(gold_answer) <= 3 and len(gold_entities) == 1:
num_2hop += 1
# if i > 500 and i % 500 == 0:
# print(">>> Caching query dict... <<< ")
# self.cache_similarity_query()
# self.cache_relation_paths()
print("* For {}".format(corpus.name))
print('* Cover ratio in all questions: {:.2f}'.format(
num_cover['all'] / len(corpus)))
print('* Cover ratio in single-entity questions: {:.2f}'.format(
num_cover['2hop'] / num_2hop))
print('* Averaged candidates per question: {:.2f}'.format(num_answers /
len(corpus)))
return corpus
def cache_similarity_query(self):
with open(self._similarity_dict_path, 'wb') as f:
pickle.dump(self._seqPair2similarity, f)
def cache_relation_paths(self):
with open(self._relation_paths_dict_path, 'wb') as f:
pickle.dump(self._entity2relations, f)