def best_titles(claim="", edocs=edict(), best=5, model=None): t2phrases = find_titles_in_claim(claim, edocs) tscores = list() for title in t2phrases: tscores.append( (title, score_title(t2phrases[title], title, claim, model))) tscores = sorted(tscores, key=lambda x: -1 * x[1])[:best] return tscores
def title_edict(t2jnum={}): edocs = edict() for title in t2jnum: l_txt = normalize_title(title) if len(l_txt) > 0: if edocs[l_txt][0] is None: edocs[l_txt] = [] edocs[l_txt][0].append(title) return edocs
def doc_ret(data=list(), edocs=edict(), best=5, model=None): ''' 对每个 claim 返回 best 个得分最高的文档标题 ''' docs = dict() for example in tqdm(data): tscores = best_titles(example['claim'], edocs, best, model) docs[example['id']] = tscores return docs
def doc_ir(data=list(), edocs=edict(), best=5, model=None): """ Returns a dictionary of n best document titles for each claim. """ docs = dict() # data = data[0] # This line for small dataset only for example in tqdm(data): tscores = best_titles(example["claim"], edocs, best, model) docs[example["id"]] = tscores return docs
def find_titles_in_claim(claim="", edocs=edict()): find = pdict(edocs) docset = {} ctoks = word_tokenize(claim) for word in ctoks: for dlist, phrase, start in find[word]: for d in dlist: if d not in docset: docset[d] = [] docset[d].append((phrase, start)) return docset
def title_edict(t2jnum={}): ''' 建立文档标题的字典 ''' edocs = edict() for title in t2jnum: _title = normalize_title(title) if len(_title) > 0: if edocs[_title][0] is None: edocs[_title] = [] edocs[_title][0].append(title) return edocs
def find_titles_in_claim(claim='', edocs=edict()): ''' 在 claim 中寻找标题短语及其在 claim 中的位置 ''' find = pdict(edocs) docset = {} # 对 claim 进行分词 ctoks = word_tokenize(claim) for word in ctoks: for dlist, phrase, start in find[word]: for d in dlist: if d not in docset: docset[d] = [] docset[d].append((phrase, start)) return docset
def best_titles(claim='', edocs=edict(), best=5, model=None): ''' 计算在 claim 中出现的得分最高(前 best 个)的文档 ''' # 在 claim 中寻找文档标题和相关句子 t2phrases = find_titles_in_claim(claim, edocs) tscores = list() # 对每个标题打分 for title in t2phrases: tscores.append( (title, score_title(t2phrases[title], title, claim, model))) # 对得分进行排序并取前 best 个 tscores = sorted(tscores, key=lambda x: -1 * x[1])[:best] return tscores
def doc_ir(data=list(), edocs=edict(), best=5, model=None): """ Returns a dictionary of n best document titles for each claim. """ rdocs = dict() for example in tqdm(data): claim = example["claim"] titles = find_titles_in_claim(claim, edocs) ctoks = word_tokenize(claim.lower()) rdocs[example["id"]] = (titles, ctoks) t2tf = titles_to_tf() doctf = load_doc_tf(rdocs, t2tf) docs = dict() for example in tqdm(data): titles, ctoks = rdocs[example["id"]] tscores = best_titles(example["claim"], ctoks, titles, doctf, best, model) docs[example["id"]] = tscores return docs