def run_gold_lm_ap():
claims, val = train_split()
top_k = 50
print("Building lms")
claim_lms = build_gold_claim_lm_train()
print("Predicting")
pred = predict_by_lm(claim_lms, claims, top_k)
print(evaluate_map(pred))
def run_a_relevant_lm():
claims, val = train_split()
top_k = 50
print("Building lms")
claim_lms = get_train_passage_a_lms()
print("Predicting")
pred = predict_by_lm(claim_lms, claims, top_k)
print(evaluate_map(pred))
def run_baseline_lm():
claims, val = train_split()
claims = val
top_k = 50
print("Building lms")
claim_lms = build_baseline_lms(claims)
print("Predicting")
pred = predict_by_lm(claim_lms, claims, top_k)
print(evaluate_map(pred))
def map_eval_with_dict(pickle_name):
d_ids: List[int] = list(load_dev_claim_ids())
claims = get_claims_from_ids(d_ids)
print("targets", len(claims))
top_k = 50
pc_score_d = load_from_pickle(pickle_name)
pred = predict_from_dict(pc_score_d, claims, top_k)
print(evaluate_map(pred))
def eval_map(split, score_d: Dict[CPIDPair, float], debug=False):
# load pre-computed perspectives
candidates: List[Tuple[int, List[Dict]]] = get_eval_candidates_from_pickle(
split)
# only evalaute what's available
valid_cids: Set[int] = set(left(score_d.keys()))
sub_candidates: List[Tuple[int, List[Dict]]] = lfilter(
lambda x: x[0] in valid_cids, candidates)
print("{} claims are evaluated".format(len(sub_candidates)))
print(left(sub_candidates))
predictions = predict_from_dict(score_d, sub_candidates, 50)
return evaluate_map(predictions, debug)
def eval_claim_lm_list(candidates: List[Tuple[int, List[Dict]]],
claim_lm_list):
def rank(e: Tuple[int, List[Dict]]):
cid, p_list = e
scored_p_list: List[Dict] = []
for p in p_list:
p['score'] = scorer(cid, p['p_tokens'])
scored_p_list.append(p)
scored_p_list.sort(key=lambda x: x['score'], reverse=True)
return cid, scored_p_list
scorer: Callable[[int, List[str]], NamedNumber] = get_lm_scorer(claim_lm_list, 0.1)
predictions = lmap(rank, candidates)
return evaluate_map(predictions, False)
def run_oracle_on_candiate_map():
claims, val = train_split()
top_k = 50
pred = predict_by_oracle_on_candidate(claims, top_k)
print(evaluate_map(pred))
def run_bm25_map():
claims, val = train_split()
top_k = 50
pred = predict_by_bm25(get_bm25_module(), claims, top_k)
print(evaluate_map(pred))
def run_bm25_map():
d_ids: List[int] = list(load_dev_claim_ids())
claims = get_claims_from_ids(d_ids)
top_k = 50
pred = predict_by_bm25(get_bm25_module(), claims, top_k)
print(evaluate_map(pred))