def run_reweight(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 7 param = {'k1': 0.5} pred = predict_by_reweighter(get_bm25_module(), claims, top_k, param) print(evaluate(pred))
def eval_from_score_d(score_d: Dict[CPID, float], top_k): candidate: List[Tuple[Dict, List[Dict]]] = load_dev_candiate() dp_not_found = 0 def get_predictions( claim_and_candidate: Tuple[Dict, List[Dict]]) -> Tuple[str, List[Dict]]: claim_info, candidates = claim_and_candidate nonlocal dp_not_found for candi in candidates: cid = candi['cid'] pid = candi['pid'] cpid = CPID("{}_{}".format(cid, pid)) if cpid in score_d: candi['new_score'] = score_d[cpid] else: dp_not_found += 1 candi['new_score'] = 0.01 candi['final_score'] = candi['new_score'] + candi['score'] / 100 candi[ 'rationale'] = "final_score={} cls_score={} lucene_score={}".format( candi['final_score'], candi['new_score'], candi['score']) candidates.sort(key=lambda c: c['final_score'], reverse=True) return claim_info['cId'], candidates[:top_k] predictions = lmap(get_predictions, candidate) print("{} data points are not found in predictions".format(dp_not_found)) r = evaluate(predictions, debug=False) print(r) return r
def run_bm25_rm(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) rm_info = load_from_pickle("perspective_dev_claim_rm") top_k = 7 pred = predict_by_bm25_rm(get_bm25_module(), rm_info, claims, top_k) print(evaluate(pred))
def run_reweight(): top_k = 7 claims, val = train_split() param = {'k1': 1} target = claims[:50] pred = predict_by_reweighter(get_bm25_module(), target, top_k, param) print(param) print(evaluate(pred))
def run_rel_scorer(): claims, val = train_split() top_k = 6 target = filter_avail(val) print("targets", len(target)) pc_score_d = load_from_pickle("pc_rel_based_score_train") pred = predict_from_dict(pc_score_d, target, top_k) print(evaluate(pred))
def run_gold_lm(): claims, val = train_split() top_k = 5 print("Building lms") claim_lms: List[ClaimLM] = build_gold_claim_lm_train() print("Predicting") pred = predict_by_lm(claim_lms, claims, top_k) print(evaluate(pred))
def run_bert_baseline(): claims, val = train_split() top_k = 50 target = filter_avail(val) print("targets", len(target)) pc_score_d = load_from_pickle("pc_bert_baseline_score_d_train") pred = predict_from_dict(pc_score_d, target, top_k) print(evaluate(pred))
def run_eval_with_dict(pickle_name): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) print("targets", len(claims)) top_k = 8 pc_score_d = load_from_pickle(pickle_name) pred = predict_from_dict(pc_score_d, claims, top_k) print(evaluate(pred))
def run_eval_with_two_dict(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) print("targets", len(claims)) top_k = 7 pc_score_d = load_from_pickle("pc_bert_baseline_score_d") pc_score_d2 = load_from_pickle("pc_random_walk_based_score_d") pred = predict_from_two_dict(pc_score_d, pc_score_d2, claims, top_k) print(evaluate(pred))
def run_random_walk_score_with_weight(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 7 q_tf_replace = dict(load_from_pickle("random_walk_score_100")) q_tf_replace = dict_key_map(lambda x: int(x), q_tf_replace) bm25 = get_bm25_module() pred = pc_predict_vector_query_and_reweight(bm25, q_tf_replace, claims, top_k, {'k1': 0.5}) print(evaluate(pred))
def run_lm(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 5 q_tf_replace = dict(load_from_pickle("pc_dev_par_tf")) q_tf_replace = dict(load_from_pickle("random_walk_score_100")) bm25 = get_bm25_module() ctf = load_collection_tf() pred = predict_by_lm(q_tf_replace, ctf, bm25, claims, top_k) print(evaluate(pred))
def run_para_scorer(): claims, val = train_split() top_k = 6 target = filter_avail(val) print("targets", len(target)) score_pred_file: FileName = FileName("pc_para_D_pred") cpid_resolute_file: FileName = FileName("resolute_dict_580_606") pred = predict_by_para_scorer(score_pred_file, cpid_resolute_file, target, top_k) print(evaluate(pred))
def run_random_walk_score(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 7 q_tf_replace = dict(load_from_pickle("random_walk_score_100")) q_tf_replace = dict_key_map(lambda x: int(x), q_tf_replace) #q_tf_replace = dict(load_from_pickle("pc_dev_par_tf")) #q_tf_replace = dict(load_from_pickle("bias_random_walk_dev_plus_all")) bm25 = get_bm25_module() pred = pc_predict_from_vector_query(bm25, q_tf_replace, claims, top_k) print(evaluate(pred))
def run_baseline(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) print("targets", len(claims)) top_k = 5 score_pred_file: FileName = FileName("pc_para_D_pred_dev_11") cpid_resolute_file: FileName = FileName("resolute_dict_dev_11") # score_pred_file: FileName = FileName("pc_para_D_pred_dev") # cpid_resolute_file: FileName = FileName("resolute_dict_dev") pred = predict_by_para_scorer(score_pred_file, cpid_resolute_file, claims, top_k) print(evaluate(pred))
def run_lm2(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 5 tokenizer = PCTokenizer() tf_d = { c['cId']: Counter(nltk.tokenize.word_tokenize(c['text'])) for c in claims } bm25 = get_bm25_module() ctf = get_perspective_tf() pred = predict_by_lm(tf_d, ctf, bm25, claims, top_k) print(evaluate(pred))
def run_bm25(): claims, val = train_split() top_k = 20 pred = predict_by_bm25(get_bm25_module(), claims, top_k) print(evaluate(pred))
def run_oracle_on_candiate(): claims, val = train_split() top_k = 5 pred = predict_by_oracle_on_candidate(claims, top_k) print(evaluate(pred))
def run_baseline(): claims, val = train_split() top_k = 50 pred = predict_by_elastic_search(claims, top_k) print(evaluate(pred))
def run_bm25(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids) top_k = 7 pred = predict_by_bm25(get_bm25_module(), claims, top_k) print(evaluate(pred))
def run_next_sent(): d_ids: List[int] = list(load_dev_claim_ids()) claims = get_claims_from_ids(d_ids)[:10] top_k = 7 pred = pc_predict_by_bert_next_sent(get_bm25_module(), claims, top_k) print(evaluate(pred))