def load_dataset_by_split(split: str) -> Dict[str, List[Dict]]: all_data: Dict[str, List[Dict]] = l_to_map(ukp.load, all_topics) # split train / dev def is_train(entry: Dict) -> bool: return entry['set'] == 'train' def is_validation(entry: Dict) -> bool: return entry['set'] == 'val' def filter_train(data: List[Dict]) -> List[Dict]: return lfilter(is_train, data) def filter_validation(data: List[Dict]) -> List[Dict]: return lfilter(is_validation, data) if split == "train": train_data: Dict[str, List[Dict]] = dict_value_map(filter_train, all_data) return train_data elif split == "dev": val_data: Dict[str, List[Dict]] = dict_value_map(filter_validation, all_data) return val_data else: assert False
def predict_by_bm25_rm(bm25_module: BM25, rm_info: Dict[str, List[Tuple[str, str]]], claims, top_k) -> List[Tuple[str, List[Dict]]]: cid_to_text: Dict[int, str] = claims_to_dict(claims) tokenizer = PCTokenizer() def stem_merge(score_list: List[Tuple[str, float]]) -> Counter: c = Counter() for k, v in score_list: try: new_k = tokenizer.stemmer.stem(k) c[new_k] += v except UnicodeDecodeError: pass return c rm_info: Dict[str, List[Tuple[str, float]]] = dict_value_map(parse_float, rm_info) rm_info: Dict[str, List[Tuple[str, float]]] = dict_value_map(normalize_scores, rm_info) rm_info_c: Dict[str, Counter] = dict_value_map(stem_merge, rm_info) print(len(rm_info_c.keys())) print(len(claims)) not_found = set() def scorer(lucene_score, query_id) -> NamedNumber: claim_id, p_id = query_id.split("_") c_text = cid_to_text[int(claim_id)] p_text = perspective_getter(int(p_id)) score: NamedNumber = bm25_module.score(c_text, p_text) nclaim_id = int(claim_id) if nclaim_id in rm_info: ex_qtf = rm_info_c[nclaim_id] p_tokens = tokenizer.tokenize_stem(p_text) ex_score = bm25_module.score_inner(ex_qtf, Counter(p_tokens)) new_info = score.name + "({})".format(ex_score.name) score = NamedNumber(score + ex_score, new_info) else: not_found.add(claim_id) return score r = predict_interface(claims, top_k, scorer) print(not_found) return r
def load_clef_qrels() -> Dict[str, List[str]]: path1 = os.path.join(data_path, "CLEFeHealth2017IRtask", "assessments", "2017", "clef2017_qrels.txt") q_rel_d1 = load_qrels_flat(path1) path2 = os.path.join(data_path, "CLEFeHealth2017IRtask", "assessments", "2016", "task1.qrels") q_rel_d2 = load_qrels_flat(path2) def fn(pair_list): return list([doc_id for doc_id, score in pair_list if score > 0]) q_rel_1 = dict_value_map(fn, q_rel_d1) q_rel_2 = dict_value_map(fn, q_rel_d2) for key in q_rel_2: q_rel_1[key].extend(q_rel_2[key]) return q_rel_1
def group_by_qid_cid( predictions: List[Dict]) -> Dict[str, Dict[str, List[Dict]]]: grouped: Dict[str, List[Dict]] = group_by(predictions, lambda x: x['query'].query_id) grouped2: Dict[str, Dict[str, List[Dict]]] = \ dict_value_map(lambda x: group_by(x, lambda x: x['candidate'].id), grouped) return grouped2
def pc_predict_from_vector_query(bm25_module: BM25, q_tf_replace: Dict[int, Counter], claims, top_k) -> List[Tuple[str, List[Dict]]]: cid_to_text: Dict[int, str] = claims_to_dict(claims) found_claim = set() q_tf_replace_norm = dict_value_map(normalize_counter, q_tf_replace) c_qtf_d = {} for cid, c_text in cid_to_text.items(): c_tokens = bm25_module.tokenizer.tokenize_stem(c_text) c_qtf_d[cid] = Counter(c_tokens) def scorer(lucene_score, query_id) -> NamedNumber: nonlocal found_claim claim_id, p_id = query_id.split("_") i_claim_id = int(claim_id) if i_claim_id in q_tf_replace_norm: claim_qtf = Counter( dict_value_map(lambda x: x * 1, c_qtf_d[i_claim_id])) ex_qtf = q_tf_replace_norm[i_claim_id] ex_qtf = Counter(dict(ex_qtf.most_common(50))) qtf = ex_qtf + claim_qtf found_claim.add(i_claim_id) else: qtf = c_qtf_d[i_claim_id] p_text = perspective_getter(int(p_id)) p_tokens = bm25_module.tokenizer.tokenize_stem(p_text) score = bm25_module.score_inner(qtf, Counter(p_tokens)) return score r = predict_interface(claims, top_k, scorer) print("{} of {} found".format(len(found_claim), len(claims))) return r
def save_concat_dev(): # prediction_path = pjoin(output_path, "pc_long_seq11") prediction_path = pjoin(output_path, "pc_long_focus_1") scores: Dict[CPID, List[float]] = collect_pipeline2_score( prediction_path, "pc_rel_dev_info_all") reduced_score: Dict[CPID, float] = dict_value_map(sum, scores) save_to_pickle(reduced_score, "pc_concat_dev_score")
def save_ranked_list(prediction_path, meta_info, save_path): data = EstimatorPredictionViewer(prediction_path) q_dict = {} for entry in data: data_id = entry.get_vector('data_id')[0] scores = entry.get_vector('logits') q_id, doc_id = meta_info[data_id] if q_id not in q_dict: q_dict[q_id] = [] probs = softmax(scores) q_dict[q_id].append((doc_id, probs[1])) def add_rank( ranked_list: List[Tuple[str, float]]) -> List[Tuple[str, int, float]]: ranked_list.sort(key=lambda x: x[1], reverse=True) ranked_list = [(doc_id, rank, score) for rank, (doc_id, score) in enumerate(ranked_list)] return ranked_list q_dict_new = dict_value_map(add_rank, q_dict) write_ranked_list_from_d(q_dict_new, save_path)
def main(): file_path = sys.argv[1] top_n = int(sys.argv[2]) save_path = sys.argv[3] ranked_list_d: Dict[str, List[SimpleRankedListEntry]] = load_galago_ranked_list(file_path) def get_head(l: List): return l[:top_n] new_ranked_list = dict_value_map(get_head, ranked_list_d) write_ranked_list_from_s(new_ranked_list, save_path)
def __init__(self, candidates_dict: Dict[str, List[QCKCandidateI]], is_correct_fn, rel_ranked_list: Dict[str, List[TrecRankedListEntry]], kdp_as_sub_token=False): self.max_seq_length = 512 self.tokenizer = get_tokenizer() self.candidates_dict: Dict[str, List[QCKCandidateI]] = candidates_dict self._is_correct = is_correct_fn self.kdp_as_sub_token = kdp_as_sub_token self.kdp_score_d: Dict[str, Dict[str, float]] = dict_value_map( get_d_from_ranked_list, rel_ranked_list)
def load_all_data( ) -> Tuple[Dict[str, List[UkpDataPoint]], Dict[str, List[UkpDataPoint]]]: all_data: Dict[str, List[Dict]] = l_to_map(ukp.load, all_topics) # split train / dev def is_train(entry: Dict) -> bool: return entry['set'] == 'train' def is_validation(entry: Dict) -> bool: return entry['set'] == 'val' def filter_train(data: List[Dict]) -> List[Dict]: return lfilter(is_train, data) def filter_validation(data: List[Dict]) -> List[Dict]: return lfilter(is_validation, data) raw_train_data: Dict[str, List[Dict]] = dict_value_map(filter_train, all_data) raw_val_data: Dict[str, List[Dict]] = dict_value_map(filter_validation, all_data) def all_data_iterator() -> Iterator[Dict]: for data_list in chain(raw_train_data.values(), raw_val_data.values()): for dp in data_list: yield dp dp_id = 1 for dp in all_data_iterator(): dp['dp_id'] = dp_id dp_id += 1 def to_data_point(l: List[Dict]) -> List[UkpDataPoint]: return lmap(UkpDataPoint.from_dict, l) train_data = dict_value_map(to_data_point, raw_train_data) val_data = dict_value_map(to_data_point, raw_val_data) return train_data, val_data
def average_scores(out_entries: List[QKTokenLevelOutEntry]) -> Dict[WordAsID, np.array]: items: List[Iterable[Tuple[WordAsID, TokenScore]]] = lmap(collect_by_word_fn, out_entries) d: Dict[WordAsID, List] = defaultdict(list) for item in items: item: Iterable[Tuple[WordAsID, TokenScore]] = item for word, probs in item: d[word].append(probs) def average_per_dim(probs_list) -> np.array: return np.mean(np.array(probs_list), axis=0) out_d: Dict[WordAsID, np.array] = dict_value_map(average_per_dim, d) return out_d
def get_extended_eval_candidate_as_qck_raw( split) -> Dict[str, List[QCKCandidate]]: c: Dict[int, List[int]] = get_extended_eval_candidate(split) def convert_candidates(candidates: List[int]) -> List[QCKCandidate]: p_texts = lmap(perspective_getter, candidates) l: List[QCKCandidate] = [] for pid, text in zip(candidates, p_texts): l.append(QCKCandidate(str(pid), text)) return l c2: Dict[int, List[QCKCandidate]] = dict_value_map(convert_candidates, c) return dict_key_map(str, c2)
def load_candidate_all_passage( max_seq_length, max_passage_per_doc=10) -> Dict[str, List[QCKCandidateWToken]]: candidate_docs: Dict[str, List[SimpleRankedListEntry]] = load_bm25_best() def get_doc_id(l: List[SimpleRankedListEntry]): return list([e.doc_id for e in l]) candidate_doc_ids: Dict[str, List[str]] = dict_value_map( get_doc_id, candidate_docs) token_data: Dict[str, List[str]] = load_robust_tokens_for_predict() return load_candidate_all_passage_inner(candidate_doc_ids, token_data, max_seq_length, max_passage_per_doc)
def get_confidence_list_per_cid(info_dir, prediction_file) -> Dict[int, List[float]]: info = load_combine_info_jsons(info_dir) def logit_to_score_softmax(logit): return scipy.special.softmax(logit)[1] scores: Dict[DataID, Tuple[CPIDPair, float, float]] = collect_scores_and_confidence(prediction_file, info, logit_to_score_softmax) grouped = group_by(scores.values(), lambda x: x[0]) print("Group size:", len(grouped)) entries = group_by_cpid(grouped) cid_grouped = group_by(entries, lambda x: x[0]) verify_confidence_consistency(cid_grouped) return dict_value_map(lambda x: x[0][2], cid_grouped)
def predict_by_reweighter(bm25_module: BM25, claims, top_k, param) -> List[Tuple[str, List[Dict]]]: cid_to_text: Dict[int, str] = claims_to_dict(claims) claim_term_weight: Dict[int, Dict[str, float]] = get_claim_term_weighting( claims, param) nlp = spacy.load("en_core_web_sm") def do_stem(t: str) -> str: r = bm25_module.tokenizer.stemmer.stem(t) return r def stem_tokenize(text: str) -> Iterator[str]: for t in nlp(text): try: yield do_stem(t.text) except UnicodeDecodeError: pass def apply_stem(term_weight: Dict[str, float]) -> Dict[str, float]: return {do_stem(k): v for k, v in term_weight.items()} claim_term_weight: Dict[int, Dict[str, float]] = dict_value_map( apply_stem, claim_term_weight) def scorer(lucene_score, query_id) -> NamedNumber: claim_id, p_id = query_id.split("_") c_text = cid_to_text[int(claim_id)] p_text = perspective_getter(int(p_id)) qtf = Counter(stem_tokenize(c_text)) weight = claim_term_weight[int(claim_id)] new_qtf = Counter() for k, v in qtf.items(): try: w = weight[k] new_qtf[k] = w * v except Exception as e: print("Exception") print(e) print(k) tf = Counter(stem_tokenize(p_text)) score = bm25_module.score_inner(new_qtf, tf) return score r = predict_interface(claims, top_k, scorer) return r
def scorer(lucene_score, query_id) -> NamedNumber: nonlocal found_claim claim_id, p_id = query_id.split("_") i_claim_id = int(claim_id) if i_claim_id in q_tf_replace_norm: claim_qtf = Counter( dict_value_map(lambda x: x * 1, c_qtf_d[i_claim_id])) ex_qtf = q_tf_replace_norm[i_claim_id] ex_qtf = Counter(dict(ex_qtf.most_common(50))) qtf = ex_qtf + claim_qtf found_claim.add(i_claim_id) else: qtf = c_qtf_d[i_claim_id] p_text = perspective_getter(int(p_id)) p_tokens = bm25_module.tokenizer.tokenize_stem(p_text) score = bm25_module.score_inner(qtf, Counter(p_tokens)) return score
def eval( score_pred_file_name: FileName, cpid_resolute_file: FileName, n_way=3, ): topic = "abortion" pred_path: FilePath = pjoin(output_path, score_pred_file_name) dpid_resolute: Dict[str, DPID] = load_dpid_resolute(cpid_resolute_file) score_d: Dict[DPID, np.ndarray] = get_datapoint_score(pred_path, dpid_resolute, "avg") def argmax(arr: np.ndarray) -> int: return arr.argmax() pred_d: Dict[DPID, int] = dict_value_map(argmax, score_d) dev_labels = get_dev_labels(topic) if n_way == 2: def merge_label(e): dpid, label = e return dpid, { 0: 0, 1: 1, 2: 1, }[label] dev_labels = lmap(merge_label, dev_labels) def fetch_pred(e: Tuple[DPID, int]): dpid, label = e pred = pred_d[dpid] return pred gold_list: List[int] = right(dev_labels) pred_list: List[int] = lmap(fetch_pred, dev_labels) if n_way == 3: all_result = eval_3label(gold_list, pred_list) elif n_way == 2: all_result = eval_2label(gold_list, pred_list) else: assert False print(all_result) f1 = sum([result['f1'] for result in all_result]) / n_way print("Avg F1 : ", f1)
def make_qcknc_problem( passage_score_path: FilePath, info_path: FilePath, config_path: FilePath, split: str, save_name: str, ) -> None: candidate_dict: Dict[int, List[Dict]] = dict( get_eval_candidates_from_pickle(split)) queries: List[QCKQuery] = get_qck_queries(split) config = json.load(open(config_path, "r")) def get_pids(l: List[Dict]) -> List[str]: return lmap(lambda x: x['pid'], l) candidate_id_dict_1: Dict[int, List[str]] = dict_value_map( get_pids, candidate_dict) candidate_id_dict: Dict[str, List[str]] = dict_key_map(str, candidate_id_dict_1) all_candidate_ids = set(flatten(candidate_id_dict.values())) candidate_dict: Dict[str, QCKCandidate] = { cid: get_qck_candidate_from_candidate_id(cid) for cid in all_candidate_ids } data_id_to_info: Dict = load_combine_info_jsons(info_path, qk_convert_map) print("number of dat info ", len(data_id_to_info)) qk_result: List[Tuple[str, List[QKOutEntry]]] = collect_good_passages( data_id_to_info, passage_score_path, config) query_dict = {q.query_id: q for q in queries} payloads = qck_from_qk_results(qk_result, candidate_id_dict, query_dict, candidate_dict) out_dir = os.path.join(output_path, "cppnc") exist_or_mkdir(out_dir) save_path = os.path.join(out_dir, save_name + ".tfrecord") data_id_man = write_qck_as_tfrecord(save_path, payloads) info_save_path = os.path.join(out_dir, save_name + ".info") print("Payload size : ", len(data_id_man.id_to_info)) json.dump(data_id_man.id_to_info, open(info_save_path, "w")) print("tfrecord saved at :", save_path) print("info saved at :", info_save_path)
def __init__( self, queries: List[QCKQuery], candidates_dict: Dict[str, List[QCKCandidate]], is_correct_fn, ): self.max_seq_length = 512 self.tokenizer = get_tokenizer() def c_list_convert(l: List[QCKCandidate]): return lmap(self.get_qck_candidate_w_token, l) self.candidates_dict: Dict[str, List[QCKCandidateWToken]] = \ dict_value_map(c_list_convert, candidates_dict) self._is_correct = is_correct_fn self.queries: List[QCKQueryWToken] = lmap(self.get_qck_query_w_token, queries) print("{} insts will made for each kdp".format( self.num_insts_per_kdp()))
def show_tp(pred_file_path: str, info_file_path: str, input_type: str, score_type: str, qrel_path: str): judgments_raw: Dict[str, List[Tuple[str, int]]] = load_qrels_flat(qrel_path) judgments = dict_value_map(dict, judgments_raw) key_logit = "logits" def get_score(entry): return get_score_from_logit(score_type, entry[key_logit]) def get_label(query_id, candidate_id): judge_dict = judgments[query_id] if candidate_id in judge_dict: return judge_dict[candidate_id] else: return 0 rows = [] grouped = load_cache("ck_based_analysis") for pair_id, items in grouped.items(): query_id, kdp_id = pair_id if query_id not in judgments: continue e_list: List[Tuple[str, float]] = [] n_rel = 0 for item in items: score = get_score(item) doc_part_id = item['candidate'].id doc_id = get_doc_id(doc_part_id) e = (doc_id, score) e_list.append(e) label = bool(get_label(query_id, doc_id)) if label: if score > 0.1: row = [query_id, kdp_id, doc_part_id, score] rows.append(row) n_rel += 1 row = [len(items), n_rel] rows.append(row) print_table(rows)
def load_candidate_head_as_doc( doc_len=400) -> Dict[str, List[QCKCandidateWToken]]: top_k = 100 candidate_docs: Dict[str, List[SimpleRankedListEntry]] = load_bm25_best() print("Num queries : ", len(candidate_docs)) print("Loading robust collection tokens...", end="") data: Dict[str, List[str]] = load_robust_tokens_for_predict() print("Done") print("Total of {} docs".format(len(data))) def make_candidate(doc_id: str): tokens = data[doc_id] return QCKCandidateWToken(doc_id, "", tokens[:doc_len]) def fetch_docs( ranked_list: List[SimpleRankedListEntry] ) -> List[QCKCandidateWToken]: return list([make_candidate(e.doc_id) for e in ranked_list[:top_k]]) return dict_value_map(fetch_docs, candidate_docs)
def load_candidate_d(): candidate_docs: Dict[str, List[SimpleRankedListEntry]] = load_bm25_best() def get_doc_id(l: List[SimpleRankedListEntry]): return list([e.doc_id for e in l]) candidate_doc_ids: Dict[str, List[str]] = dict_value_map( get_doc_id, candidate_docs) # token_data: Dict[str, List[str]] = load_robust_tokens_for_predict() docs = load_from_pickle("robust04_docs_predict") out_d = {} top_k = 100 for query_id, doc_id_list in candidate_doc_ids.items(): new_entries = [] for doc_id in doc_id_list[:top_k]: # tokens = token_data[doc_id] content = docs[doc_id] new_entries.append((doc_id, content)) out_d[query_id] = new_entries return out_d
def get_qck_candidate_from_ranked_list( ranked_list) -> Dict[str, List[QCKCandidate]]: d: Dict[str, List[str]] = get_candidate_ids_from_ranked_list(ranked_list) return dict_value_map(add_texts, d)
def do_job(input_dir, output_dir, info_dir, label_info_path, max_entries, job_id): exist_or_mkdir(output_dir) info_output_dir = output_dir + "_info" exist_or_mkdir(info_output_dir) label_info: List[Tuple[str, str, int]] = json.load(open(label_info_path, "r")) label_info_d = {(str(a), str(b)): c for a, b, c in label_info} pred_path = os.path.join(input_dir, str(job_id) + ".score") #info_path = os.path.join(info_dir, str(job_id) + ".info") info_path = info_dir output_path = os.path.join(output_dir, str(job_id)) info_output_path = os.path.join(info_output_dir, str(job_id)) info = load_combine_info_jsons(info_path, qck_convert_map, True) fetch_field_list = ["vector", "data_id"] predictions = join_prediction_with_info(pred_path, info, fetch_field_list) def get_qid(entry): return entry['query'].query_id def get_candidate_id(entry): return entry['candidate'].id def pair_id(entry) -> Tuple[str, str]: return get_qid(entry), get_candidate_id(entry) groups: Dict[Tuple[str, str], List[Dict]] = group_by(predictions, pair_id) def get_new_entry(entries: List[Dict]): if not entries: return None vectors: Vectors = list([e['vector'] for e in entries]) key = pair_id(entries[0]) if key in label_info_d: label: Label = label_info_d[key] else: label: Label = 0 return vectors, label g2: Dict[Tuple[str, str], Tuple[Vectors, Label]] = dict_value_map(get_new_entry, groups) base = 100 * 1000 * job_id max_count = 100 * 1000 * (job_id + 1) data_id_manager = DataIDManager(base, max_count) def get_out_itr() -> Iterable[Tuple[int, Tuple[Vectors, Label]]]: for key, data in g2.items(): qid, cid = key data_info = { 'qid': qid, 'cid': cid, } data_id = data_id_manager.assign(data_info) yield data_id, data write_to_file(output_path, get_out_itr(), max_entries) json.dump(data_id_manager.id_to_info, open(info_output_path, "w"))
def pc_predict_vector_query_and_reweight( bm25_module: BM25, q_tf_replace: Dict[int, Counter], claims, top_k, param) -> List[Tuple[str, List[Dict]]]: cid_to_text: Dict[int, str] = claims_to_dict(claims) found_claim = set() q_tf_replace_norm = dict_value_map(normalize_counter, q_tf_replace) def do_stem(t: str) -> str: r = bm25_module.tokenizer.stemmer.stem(t) return r def apply_stem(term_weight: Dict[str, float]) -> Dict[str, float]: return {do_stem(k): v for k, v in term_weight.items()} claim_term_weight: Dict[int, Dict[str, float]] = get_claim_term_weighting( claims, param) claim_term_weight: Dict[int, Dict[str, float]] = dict_value_map( apply_stem, claim_term_weight) nlp = spacy.load("en_core_web_sm") def stem_tokenize(text: str) -> Iterator[str]: for t in nlp(text): try: yield do_stem(t.text) except UnicodeDecodeError: pass def get_qtf(claim_id): weight = claim_term_weight[claim_id] new_qtf = Counter() c_text = cid_to_text[int(claim_id)] qtf = Counter(stem_tokenize(c_text)) print(weight) for k, v in qtf.items(): try: if k in weight: w = weight[k] new_qtf[k] = w * v else: new_qtf[k] = v except Exception as e: print("Exception") print(e) print(k) return new_qtf c_qtf_d = {k: get_qtf(k) for k in cid_to_text.keys()} # for cid, c_text in cid_to_text.items(): # c_tokens = bm25_module.tokenizer.tokenize_stem(c_text) # c_qtf_d[cid] = Counter(c_tokens) def scorer(lucene_score, query_id) -> NamedNumber: nonlocal found_claim claim_id, p_id = query_id.split("_") i_claim_id = int(claim_id) if i_claim_id in q_tf_replace_norm: ex_qtf = q_tf_replace_norm[i_claim_id] ex_qtf = Counter(dict(ex_qtf.most_common(50))) qtf = ex_qtf + c_qtf_d[i_claim_id] found_claim.add(i_claim_id) else: qtf = c_qtf_d[i_claim_id] p_text = perspective_getter(int(p_id)) p_tokens = bm25_module.tokenizer.tokenize_stem(p_text) score = bm25_module.score_inner(qtf, Counter(p_tokens)) return score r = predict_interface(claims, top_k, scorer) print("{} of {} found".format(len(found_claim), len(claims))) return r
def prec_recall(pred_file_path: str, info_file_path: str, input_type: str, score_type: str, qrel_path: str): judgments_raw: Dict[str, List[Tuple[str, int]]] = load_qrels_flat(qrel_path) judgments = dict_value_map(dict, judgments_raw) grouped = load_cache("ck_based_analysis") key_logit = "logits" if grouped is None: f_handler = get_format_handler(input_type) info: Dict = load_combine_info_jsons(info_file_path, f_handler.get_mapping(), f_handler.drop_kdp()) data: List[Dict] = join_prediction_with_info(pred_file_path, info, ["data_id", key_logit]) grouped = group_by(data, get_qk_pair_id) def get_score(entry): return get_score_from_logit(score_type, entry[key_logit]) def get_label(query_id, candidate_id): judge_dict = judgments[query_id] if candidate_id in judge_dict: return judge_dict[candidate_id] else: return 0 head = [ "query_id", "kdp_id", "accuracy", "precision", "recall", "f1", "tp", "fp", "tn", "fn" ] rows = [head] for pair_id, items in grouped.items(): query_id, kdp_id = pair_id if query_id not in judgments: continue e_list: List[Tuple[str, float]] = [] labels = [] predictions = [] for item in items: score = get_score(item) doc_part_id = item['candidate'].id doc_id = get_doc_id(doc_part_id) e = (doc_id, score) e_list.append(e) label = bool(get_label(query_id, doc_id)) labels.append(label) prediction = score > 0.5 predictions.append(prediction) scores = get_acc_prec_recall(predictions, labels) row = [ query_id, kdp_id, scores['accuracy'], scores['precision'], scores['recall'], scores['f1'], scores['tp'], scores['fp'], scores['tn'], scores['fn'] ] rows.append(row) print_table(rows)
def first_pid_as_rep() -> Dict[int, List[int]]: id_dict: Dict[int, List[List[int]]] = get_claim_perspective_id_dict() id_dict_small: Dict[int, List[int]] = dict_value_map( lambda ll: lmap(lambda l: l[0], ll), id_dict) return id_dict_small
def get_docs_from_q_res_path_top_k(file_path, top_k) -> Dict[str, List[List[str]]]: ranked_list_d: Dict[str, List[SimpleRankedListEntry]] = load_galago_ranked_list(file_path) ranked_list_d = dict_value_map(lambda x:x[:top_k], ranked_list_d) return get_docs_from_q_res(ranked_list_d)
def get_docs_from_q_res(ranked_list_d: Dict[str, List[SimpleRankedListEntry]]) -> Dict[str, List[List[str]]]: print(len(ranked_list_d)) return dict_value_map(get_docs_from_ranked_list, ranked_list_d)
def pc_predict_to_inspect(bm25_module: BM25, q_tf_replace: Dict[int, Counter], q_tf_replace_0: Dict[int, Counter], claims, top_k): gold = get_claim_perspective_id_dict() q_tf_replace_norm = dict_value_map(normalize_counter, q_tf_replace) q_tf_replace_0_norm = dict_value_map(normalize_counter, q_tf_replace_0) cid_to_text: Dict[int, str] = claims_to_dict(claims) c_qtf_d = {} for cid, c_text in cid_to_text.items(): c_tokens = bm25_module.tokenizer.tokenize_stem(c_text) c_qtf_d[cid] = Counter(c_tokens) def counter_to_str(c: Dict) -> str: s = "" for k, v in c.items(): s += "{0} {1:.2f}".format(k, v) + "\t" return s for claim in claims: cid = claim['cId'] i_claim_id = int(cid) claim_text = claim['text'] lucene_results = es_helper.get_perspective_from_pool(claim_text, 50) candidate_pids = [] for rank, (_text, _pid, _score) in enumerate(lucene_results): candidate_pids.append(_pid) if i_claim_id in q_tf_replace_norm: claim_qtf = Counter( dict_value_map(lambda x: x * 1, c_qtf_d[i_claim_id])) ex_qtf = q_tf_replace_norm[i_claim_id] ex_qtf = Counter(dict(ex_qtf.most_common(50))) qtf = ex_qtf + claim_qtf else: qtf = c_qtf_d[i_claim_id] ranked_list = [] for pid in candidate_pids: p_text = perspective_getter(int(pid)) p_tokens = bm25_module.tokenizer.tokenize_stem(p_text) score = bm25_module.score_inner(qtf, Counter(p_tokens)) debug_str = "" e = score, pid, p_text, debug_str ranked_list.append(e) gold_pids = gold[cid] def is_correct(pid): for pids in gold_pids: if pid in pids: return True return False ranked_list.sort(key=lambda x: x[0], reverse=True) qtf_idf_applied = { k: v * bm25_module.term_idf_factor(k) for k, v in qtf.items() } print() print("Claim: ", cid, claim_text) for cluster in gold_pids: print("-") for pid in cluster: print(pid, perspective_getter(pid)) print() print("qtf:", counter_to_str(qtf)) if i_claim_id in q_tf_replace_norm and i_claim_id in q_tf_replace_0_norm: print("ex_qtf:", counter_to_str(ex_qtf)) ex_qtf_0 = q_tf_replace_0_norm[i_claim_id] ex_qtf_0 = Counter(dict(ex_qtf_0.most_common(50))) print("ex_qtf_0:", counter_to_str(ex_qtf_0)) print("qtf idf apllied:", counter_to_str(qtf_idf_applied)) for score, pid, p_text, debug_str in ranked_list[:top_k]: if i_claim_id in q_tf_replace_0_norm: p_text = perspective_getter(int(pid)) p_tokens = bm25_module.tokenizer.tokenize_stem(p_text) ex_qtf_0 = q_tf_replace_0_norm[i_claim_id] qtf = ex_qtf_0 + c_qtf_d[i_claim_id] score2 = bm25_module.score_inner(qtf, Counter(p_tokens)) correct_str = "Y" if is_correct(pid) else "N" print("{0} {1:.2f} ({2:.2f}) {3} / {4} / {5}".format( correct_str, score, score2, p_text, score.name, score2.name))