def __calculate_avg_doc_length(self): papers = self.get_all_paper() intro, background, methods, result, discussion, overall = [], [], [], [], [], [] for paper in papers: overall.append(paper.word_hist.number_of_words()) intro.append( sections_to_word_hist( paper.get_introduction()).number_of_words()) background.append( sections_to_word_hist( paper.get_background()).number_of_words()) methods.append( sections_to_word_hist(paper.get_methods()).number_of_words()) result.append( sections_to_word_hist(paper.get_results()).number_of_words()) discussion.append( sections_to_word_hist( paper.get_discussion()).number_of_words()) return { "whole-document": mean(overall, True), IMRaDType.INTRODUCTION.name: mean(intro, True), IMRaDType.BACKGROUND.name: mean(background, True), IMRaDType.METHODS.name: mean(methods, True), IMRaDType.RESULTS.name: mean(result, True), IMRaDType.DISCUSSION.name: mean(discussion, True) }
def get_ranking(paper, queries, settings, api): tfidf = {} df = settings["df"] N = settings["N"] for imrad, query in queries.items(): if imrad == "whole-document": hist = paper.word_hist else: sections = paper.get_sections_with_imrad_type(imrad) hist = sections_to_word_hist(sections) key_values = {} for querie_word in query.split(): df_val = df[imrad][querie_word] # query word is in no other paper -> protect against dividing by 0 if not df_val: continue tf_val = hist.get_tf(querie_word) idf_val = math.log10(N / df_val) tfidf_val = tf_val * idf_val key_values[querie_word] = {"tfidf": tfidf_val, "tf": tf_val, "idf": idf_val} tfidf[imrad] = {"sumwords": sum(hist.values()), "keys": key_values, "score": sum([val["tfidf"] for val in key_values.values()])} return sum([rating["score"] for rating in tfidf.values()]), tfidf
def __calculate_all_papers_hist(self): papers = self.get_all_paper() intro, background, methods, result, discussion, overall = WordHist( ), WordHist(), WordHist(), WordHist(), WordHist(), WordHist() for paper in papers: overall.append(paper.get_combined_word_hist()) intro.append(sections_to_word_hist(paper.get_introduction())) background.append(sections_to_word_hist(paper.get_background())) methods.append(sections_to_word_hist(paper.get_methods())) result.append(sections_to_word_hist(paper.get_results())) discussion.append(sections_to_word_hist(paper.get_discussion())) return { "whole-document": overall, IMRaDType.INTRODUCTION.name: intro, IMRaDType.BACKGROUND.name: background, IMRaDType.METHODS.name: methods, IMRaDType.RESULTS.name: result, IMRaDType.DISCUSSION.name: discussion }
def get_ignoring_keys(paper, queries, importance_sections): if not importance_sections: return paper.word_hist.query_to_keys(queries["whole-document"]) ignored, sections = [], [] for imrad, query in queries.items(): if imrad == "whole-document": continue sections = paper.get_sections_with_imrad_type(imrad) ignored.extend(sections_to_word_hist(sections).query_to_keys(query)) return ignored
def get_ranking(paper, queries, settings, api): dfr = {} avg_doc_length_dict = api.client.get_avg_doc_length() all_papers_hist_dict = api.client.get_all_papers_hist() N = api.client.get_number_of_papers() for imrad, query in queries.items(): if imrad == "whole-document": hist = paper.word_hist else: sections = paper.get_sections_with_imrad_type(imrad) hist = sections_to_word_hist(sections) # + 1... avoid dividing by zero doc_length = hist.number_of_words() + 1 avg_doc_length = avg_doc_length_dict[imrad] all_papers_hist = all_papers_hist_dict[imrad] key_values = {} for querie_word in query.split(): # f_ij == term frequency f_ij_norm = hist.get_tf(querie_word) * (avg_doc_length / doc_length) if f_ij_norm == 0 or querie_word not in all_papers_hist: continue pi_i = all_papers_hist[querie_word] / N p_kic = f_ij_norm * math.log10(f_ij_norm / pi_i) + ( pi_i + (1 / (12 * f_ij_norm + 1)) - f_ij_norm) * math.log10( math.e) + 0.5 * math.log(2 * math.pi * f_ij_norm) p_kidi = 1 / (f_ij_norm + 1) w_ij = p_kic * p_kidi f_iq = query.count(querie_word) rank = f_iq * w_ij key_values[querie_word] = { "rank": rank, "f_iq": f_iq, "f_ij_norm": f_ij_norm, "pi_i": pi_i, "p_kic": p_kic, "p_kidi": p_kidi } dfr[imrad] = { "sumwords": sum(hist.values()), "keys": key_values, "score": sum([val["rank"] for val in key_values.values()]) } return sum([rating["score"] for rating in dfr.values()]), dfr
def __create_hists(queries, papers): hists = {} for imrad, query in queries.items(): hists[imrad] = {} for paper in papers: if imrad == "whole-document": hist = paper.word_hist else: sections = paper.get_sections_with_imrad_type(imrad) hist = sections_to_word_hist(sections) hists[imrad][paper.id] = hist return hists
def get_ranking(paper, queries, settings, api): k1 = copy.deepcopy(settings.get("k1")) b = copy.deepcopy(settings.get("b")) bm25 = {} df = settings["df"] N = settings["N"] avg_doc_length_dict = api.client.get_avg_doc_length() for imrad, query in queries.items(): if imrad == "whole-document": hist = paper.word_hist else: sections = paper.get_sections_with_imrad_type(imrad) hist = sections_to_word_hist(sections) doc_length = hist.number_of_words() + 1 avg_doc_length = avg_doc_length_dict[imrad] key_values = {} for querie_word in query.split(): df_val = df[imrad][querie_word] # query word is in no other paper -> protect against dividing by 0 if not df_val: continue tf_val = hist.get_tf(querie_word) idf_val = math.log10((N - df_val + 0.5) / (df_val + 0.5)) b25_val = idf_val * ((tf_val * (k1 + 1)) / (tf_val + k1 * (1 - b + b * (doc_length / avg_doc_length)))) key_values[querie_word] = {"bm25": b25_val, "tf": tf_val, "idf": idf_val} bm25[imrad] = {"sumwords": sum(hist.values()), "keys": key_values, "score": sum([val["bm25"] for val in key_values.values()])} return sum([rating["score"] for rating in bm25.values()]), bm25
def get_ranking(paper, queries, settings, api): info = {} word_value = [] ranking = 0.0 for imrad_type, query in queries.items(): if imrad_type == "whole-document": hist = paper.word_hist else: sections = paper.get_sections_with_imrad_type(imrad_type) hist = sections_to_word_hist(sections) for word in query.split(): rank = hist.get_tf(word) ranking += rank word_value.append([word, rank]) info[imrad_type] = { "rank": ranking, "sumwords": sum(hist.values()), "keyvalues": word_value } return sum([ranking["rank"] for ranking in info.values()]), info