def train_mention_pos_equal_position(): question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") train_question_posword = read_posques_posword( "../data/cluster/train.quespos.posword") train_question_fnentity = read_ques_fn_entity( "../data/cluster/train.question.friendlyname.entity") # test_question_fnentity = read_ques_fn_entity("../data/test/test.question.friendlyname.entity") ques_position_pos_equal = mention_position_pos_equal( question_posword, train_question_posword, train_question_fnentity) return ques_position_pos_equal
def generate_question_entity_pro_relation_words(): entity_match = conquer() question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") question_entity_pro_relation_words_concerned = dict() for ques_fnentity in entity_match: phrases = entity_match[ques_fnentity] ques = ques_fnentity.split("###")[0] posword = question_posword[ques] word_list = posword_wordlist(posword) for phrase in phrases: if (phrase != "on") & (phrase != "the") & (phrase != "of"): words_phrase = phrase.split(" ") # for word_phrase in set(words_phrase): # if word_phrase in word_list: # word_list.remove(word_phrase) word_relation_concerned = set(word_list) entity_pros = phrases[phrase] for entity in entity_pros: pro = entity_pros[entity] question_entity_pro = "###".join([ques, entity, str(pro)]) question_entity_pro_relation_words_concerned[ question_entity_pro] = word_relation_concerned write_dict( question_entity_pro_relation_words_concerned, "../data/relation/test.easy.partial.question_entity_pro_relation_words" ) return question_entity_pro_relation_words_concerned
def easyquestion_friendlyname_pos_entity(): easyquestion_friendlyname_pos_entity = dict() easyquestion_posword = read_posques_posword( "..\\data\\test\\test.easy.quespos.posword") question_fnentity = read_ques_fn_entity( "..\\data\\test\\test.question.friendlyname.entity") pos_contained_set = set() pos_combination_set = set() for ques in easyquestion_posword: fnentity = question_fnentity[ques] # if(len(fnentity)!=1): # print(ques+"\t"+fnentity) posword = easyquestion_posword[ques] fnentity_word_pos = fnentity[0] # print(fnentity_word_pos) pos_ques = "" pos_combination = "" for pos_word in posword: pos = pos_word.split("\t")[0] pos_ques = pos_ques + pos + "\t" word = pos_word.split("\t")[1] if word in fnentity[0]: fnentity_word_pos = fnentity_word_pos + "\t" + word + "\t" + pos + "###" pos_contained_set.add(pos) pos_combination = pos_combination + pos + "\t" pos_combination_set.add(pos_combination) easyquestion_friendlyname_pos_entity[ques + "###" + pos_ques] = fnentity_word_pos write_dict_str(easyquestion_friendlyname_pos_entity, "..\\data\\test\\test.easyquespos.friename.wordpos") print(pos_contained_set) print(pos_combination_set)
def question_friendlyname_pos_entity(): question_friendlyname_pos_entity=dict() question_posword = read_posques_posword("..\\data\\cluster\\quespos_posword") question_fnentity=read_ques_fn_entity("..\\data\\cluster\\train.question.friendlyname.entity") pos_contained_set=set() pos_combination_set=set() for ques in question_posword: fnentity=question_fnentity[ques] # if(len(fnentity)!=1): # print(ques+"\t"+fnentity) posword=question_posword[ques] fnentity_word_pos_list=list() for fnentity_one in fnentity: pos_ques = "" fnentity_word_pos=fnentity_one # print(fnentity_word_pos) pos_combination="" for pos_word in posword: pos=pos_word.split("\t")[0] pos_ques=pos_ques+pos+"\t" word=pos_word.split("\t")[1] if word in fnentity_one: fnentity_word_pos=fnentity_word_pos+"\t"+word+"\t"+pos+"###" pos_contained_set.add(pos) pos_combination=pos_combination+pos+"\t" pos_combination_set.add(pos_combination) fnentity_word_pos_list.append(fnentity_word_pos) question_friendlyname_pos_entity[ques+"###"+pos_ques]=fnentity_word_pos_list write_dict(question_friendlyname_pos_entity,"..\\data\\cluster\\train.easyquespos.friename.wordpos") print(pos_contained_set) print(pos_combination_set)
def question_friendlynamejinsuo_pos_entity(): # question_posword = read_posques_posword("Users\\lanlanzh\\kbqa_python\\data\\test\\test.easy.quespos.posword") question_posword = read_posques_posword("../data/cluster/train.quespos.posword") question_fnentity=read_ques_fn_entity("../data/cluster/train.question.friendlyname.entity") mention_map_pos_com=set() for ques in question_posword: fnentity=question_fnentity[ques] posword=question_posword[ques] fnentity_word_pos_list=list() for fnentity_one in fnentity: hit=False friendlyname=fnentity_one.split("\t")[0] entity=fnentity_one.split("\t")[1] friendlyname_jinsuo=friendlyname.replace(" ","") size_posword=len(posword) for i in range(0,size_posword): pos_comb="" word_comb="" pos=posword[i].split("\t")[0] word=posword[i].split("\t")[1] if word == friendlyname_jinsuo: hit=True mention_map_pos_com.add(pos) elif word == friendlyname_jinsuo+"s": hit=True mention_map_pos_com.add(pos) elif word == friendlyname_jinsuo+".": hit=True mention_map_pos_com.add(pos) elif word in friendlyname_jinsuo: pos_comb=pos_comb+pos+"\t" word_comb=word_comb+word for j in range(i+1,size_posword): # print("word_comb"+word_comb) pos_j = posword[j].split("\t")[0] word_j = posword[j].split("\t")[1] word_comb=word_comb+word_j # print("word_comb" + word_comb) pos_comb=pos_comb+pos_j+"\t" if word_comb==friendlyname_jinsuo: hit=True mention_map_pos_com.add(pos_comb) break elif word_comb==friendlyname_jinsuo+"s": hit=True mention_map_pos_com.add(pos_comb) break elif word_comb==friendlyname_jinsuo+".": hit=True mention_map_pos_com.add(pos_comb) break elif word_comb not in friendlyname_jinsuo: break if hit==False: print(fnentity_one+"\t"+ques) print(posword) write_set(mention_map_pos_com,"../data/cluster/train.mention.pos.scomposition") return mention_map_pos_com
def hit_by_np(): np_mention_indexrange = read_dict_mention_indexrange( "..\\data\\test\\test.easy.ques.np.index.range") test_easy_question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") ques_indexrange_entity_pros = dict() ques_entity_pros = dict() # phrase_num=dict() for ques in np_mention_indexrange: print(ques) posword = test_easy_question_posword[ques] wordlist = posword_wordlist(posword) mention_indexranges = np_mention_indexrange[ques] #n>=3 n_indexrange = genenrate_indexrange(wordlist) np_n_indexrange = set(mention_indexranges) | n_indexrange indexrange_phrase = position_to_phrases(np_n_indexrange, wordlist) indexrange_entity_pros = dict() for indexrange in indexrange_phrase: phrase = indexrange_phrase[indexrange] entity_pros_friendlyname = friendlyname_entity_match_one(phrase) entity_pros_alias = aliases_entity_match_one(phrase) entity_pros_name = name_entity_match_one(phrase) entity_pros_clueweb = clueweb_entity_match_one(phrase) entity_pros_sum = entity_pros_sum_all(entity_pros_friendlyname, entity_pros_alias, entity_pros_name, entity_pros_clueweb) # if (ques == "what is needed to prepare cuba libre?"): # print(mention_indexranges) # print(wordlist) # print(indexrange) # print("yeah\t", phrase) # print("yeah\t", entity_pros_friendlyname) # print("yeah\t", entity_pros_alias) # print("yeah\t", entity_pros_name) # print("yeah\t", entity_pros_clueweb) indexrange_entity_pros[indexrange] = entity_pros_sum ques_indexrange_entity_pros[ques] = indexrange_entity_pros entity_pros_per_ques = entity_pros_per_question(indexrange_entity_pros) ques_entity_pros[ques] = entity_pros_per_ques write_dict_dict(ques_entity_pros, "..\\data\\test\\test.easy.ques.np_entitymatch") write_dict_dict_dict( ques_indexrange_entity_pros, "..\\data\\test\\test.easy.ques.np_ques_indexrange_entity_pros") return ques_entity_pros
def train_frname_in_test_position(): ques_frname_in_position = dict() question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") train_question_fnentity = read_ques_fn_entity( "../data/cluster/train.question.friendlyname.entity") fnentitys = set() for train_ques in train_question_fnentity: fnentity = train_question_fnentity[train_ques] fnentitys = fnentitys | set(fnentity) for ques in question_posword: posword = question_posword[ques] position_i_j = train_frname_position(fnentitys, posword) ques_frname_in_position[ques] = position_i_j return ques_frname_in_position
def conquer(): # question_posword=read_posques_posword("..\\data\\test\\test.quespos.posword") question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") entity_match = dict() for question in question_posword: print(question) posword = question_posword[question] print(posword) phrasen_entity_pro = entity_link_rein(posword) # phrasen_entity_pro=entity_link(posword) if len(phrasen_entity_pro) != 0: entity_match[question] = phrasen_entity_pro #break return entity_match
def match_by_friendlyname(): question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") test_question_fnentity = read_ques_fn_entity( "../data/test/test.question.friendlyname.entity") entity_match = dict() for question in question_posword: fnentity_test = test_question_fnentity[question] posword = question_posword[question] word_list = posword_wordlist(posword) phrases = combine_wordlist(word_list) phrase_frnentity = friendlyname_entity_match(phrases) if len(phrase_frnentity) > 0: # a=1 entity_match[question + "###" + "\t".join(fnentity_test)] = phrase_frnentity return entity_match
def question_word_frequency(): question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") word_num = dict() for ques in question_posword: posword = question_posword[ques] word_list = posword_wordlist(posword) for word in set(word_list): if word in word_num: num = word_num[word] num += 1 word_num[word] = num else: word_num[word] = 1 word_num = dict(sorted(word_num.items(), key=lambda d: d[1], reverse=True)) for word in word_num: print(word, "\t", word_num[word]) return word_num
def test_easy_mention_position(): question_posword = read_posques_posword( "..\\data\\test\\test.easy.quespos.posword") question_fnentity = read_ques_fn_entity( "..\\data\\test\\test.question.friendlyname.entity") position_question_posword = dict() for ques in question_posword: position = list() fnentity = question_fnentity[ques] posword = question_posword[ques] if len(fnentity) != 1: print(ques) for fnentity_one in fnentity: hit = False friendlyname = fnentity_one.split("\t")[0] entity = fnentity_one.split("\t")[1] friendlyname_jinsuo = friendlyname.replace(" ", "") friendlyname_jinsuos = friendlyname_jinsuo + "s" friendlyname_jinsuodot = friendlyname_jinsuo + "." size_posword = len(posword) for i in range(0, size_posword): pos_comb = "" word_comb = "" pos = posword[i].split("\t")[0] word = posword[i].split("\t")[1] word = word.replace("`", "'") if (word == friendlyname_jinsuo) | ( word == friendlyname_jinsuos) | ( word == friendlyname_jinsuodot): hit = True position.append(str(i)) elif (word in friendlyname_jinsuo) | ( word in friendlyname_jinsuos) | ( word in friendlyname_jinsuodot): pos_comb = pos_comb + pos + "\t" word_comb = word_comb + word for j in range(i + 1, size_posword): # print("word_comb"+word_comb) pos_j = posword[j].split("\t")[0] word_j = posword[j].split("\t")[1] word_j = word_j.replace("`", "'") word_comb = word_comb + word_j # print("word_comb" + word_comb) pos_comb = pos_comb + pos_j + "\t" if (word_comb == friendlyname_jinsuo) | ( word_comb == friendlyname_jinsuos) | ( word_comb == friendlyname_jinsuodot): hit = True position.append("\t".join([str(i), str(j)])) break elif (word_comb not in friendlyname_jinsuo) & ( word_comb not in friendlyname_jinsuos) & ( word_comb not in friendlyname_jinsuodot): break if hit == False: print(friendlyname) print(ques) if len(position) != 1: print(position) print(friendlyname) print(ques) if position[0] in position_question_posword: question_poswords = position_question_posword[position[0]] question_poswords.add(ques + "###" + "\t".join(question_posword[ques])) position_question_posword[position[0]] = question_poswords else: question_poswords = set() question_poswords.add(ques + "###" + "\t".join(question_posword[ques])) position_question_posword[position[0]] = question_poswords position_question_posword_sort = dict( sorted(position_question_posword.items(), key=lambda d: len(d[1]), reverse=True)) # write_dict(position_question_posword_sort, "..\\data\\test\\test.easy.position.mention") return position_question_posword
def conquer(): # question_posword=read_posques_posword("..\\data\\test\\test.quespos.posword") question_posword = read_posques_posword( "../data/test/test.easy.quespos.posword") train_question_posword = read_posques_posword( "../data/cluster/train.quespos.posword") train_question_fnentity = read_ques_fn_entity( "../data/cluster/train.question.friendlyname.entity") test_question_fnentity = read_ques_fn_entity( "../data/test/test.question.friendlyname.entity") entity_match = dict() test_pos_equal_mention = mention_pos_equal(question_posword, train_question_posword, train_question_fnentity) test_pos_similar_mention = mention_pos_similar(question_posword, train_question_posword, train_question_fnentity) for question in question_posword: fnentity_test = test_question_fnentity[question] posword = question_posword[question] word_list = posword_wordlist(posword) phrases = combine_wordlist(word_list) phrase_frnentity = friendlyname_entity_match(phrases) if len(phrase_frnentity) > 0: #a=1 entity_match[question + "###" + "\t".join(fnentity_test)] = phrase_frnentity elif question in test_pos_equal_mention: # a = 1 mention_possible = test_pos_equal_mention[question] pos_equal_phrase_entityall = dict() pos_equal_phrase_frnentity = friendlyname_entity_match( mention_possible) pos_equal_phrase_aliasentity = aliases_entity_match( mention_possible) pos_equal_phrase_nameentity = name_entity_match(mention_possible) pos_equal_phrase_cluewebentity = clueweb_entity_match( mention_possible) pos_equal_phrase_entityall = add_dict_dict( pos_equal_phrase_entityall, pos_equal_phrase_frnentity) pos_equal_phrase_entityall = add_dict_dict( pos_equal_phrase_entityall, pos_equal_phrase_aliasentity) pos_equal_phrase_entityall = add_dict_dict( pos_equal_phrase_entityall, pos_equal_phrase_nameentity) pos_equal_phrase_entityall = add_dict_dict( pos_equal_phrase_entityall, pos_equal_phrase_cluewebentity) entity_match[question + "###" + "\t".join(fnentity_test)] = pos_equal_phrase_entityall elif question in test_pos_similar_mention: mention_similar_possible = test_pos_similar_mention[question] pos_similar_phrase_entityall = dict() pos_similar_phrase_frnentity = friendlyname_entity_match( mention_similar_possible) pos_similar_phrase_aliasentity = aliases_entity_match( mention_similar_possible) pos_similar_phrase_nameentity = name_entity_match( mention_similar_possible) pos_similar_phrase_cluewebentity = clueweb_entity_match( mention_similar_possible) pos_similar_phrase_entityall = add_dict_dict( pos_similar_phrase_entityall, pos_similar_phrase_frnentity) pos_similar_phrase_entityall = add_dict_dict( pos_similar_phrase_entityall, pos_similar_phrase_aliasentity) pos_similar_phrase_entityall = add_dict_dict( pos_similar_phrase_entityall, pos_similar_phrase_nameentity) pos_similar_phrase_entityall = add_dict_dict( pos_similar_phrase_entityall, pos_similar_phrase_cluewebentity) entity_match[ question + "###" + "\t".join(fnentity_test)] = pos_similar_phrase_entityall for ques in entity_match: for phrase in entity_match[ques]: entity_pros = entity_match[ques][phrase] entity_pros = dict( sorted(entity_pros.items(), key=lambda d: d[1], reverse=True)) entity_match[ques][phrase] = entity_pros print(len(entity_match)) return entity_match