def main():
claim_text_d: Dict[int, str] = get_all_claim_d()
claim_text_d: Dict[str, str] = dict_key_map(str, claim_text_d)
evi_dict: Dict[str, str] = dict_key_map(str, load_evidence_dict())
evi_gold_dict: Dict[str, List[int]] = evidence_gold_dict_str_qid()
print("V2")
def print_entry(entry):
evidence_text = evi_dict[entry.doc_id]
print("[{}] {}: {}".format(entry.rank, entry.doc_id, evidence_text))
ranked_list_dict = load_ranked_list_grouped(sys.argv[1])
for query, ranked_list in ranked_list_dict.items():
print()
claim_id, perspective_id = query.split("_")
gold_ids: List[str] = lmap(str, evi_gold_dict[query])
if not gold_ids:
print("query {} has no gold".format(query))
continue
assert gold_ids
claim_text = claim_text_d[claim_id]
perspective_text = perspective_getter(int(perspective_id))
pos_entries = []
neg_entries = []
for entry in ranked_list:
label = entry.doc_id in gold_ids
if label:
pos_entries.append(entry)
elif entry.rank < 3:
neg_entries.append(entry)
if not pos_entries:
print("gold not in ranked list")
continue
num_rel = len(pos_entries)
correctness = []
for entry in ranked_list[:num_rel]:
label = entry.doc_id in gold_ids
correctness.append(int(label))
precision = average(correctness)
if precision > 0.99:
print("Good")
continue
print("precision at {}: {}".format(num_rel, precision))
print("Claim: ", claim_text)
print("perspective_text: ", perspective_text)
print(" < GOLD >")
foreach(print_entry, pos_entries)
print(" < False Positive >")
foreach(print_entry, neg_entries)
def get_idf_keyword_score(problems: List[QueryDoc],
get_idf) -> Iterable[Counter]:
stemmer = CacheStemmer()
ticker = TimeEstimator(len(problems))
for p in problems:
tokens = p.doc
tf = Counter()
reverse_map = {} # Stemmed -> raw
tokens = [t for t in tokens if t not in [".", ",", "!"]]
for raw_t in tokens:
stem_t = stemmer.stem(raw_t)
reverse_map[stem_t] = raw_t
tf[stem_t] += 1
score_d = Counter()
for term, cnt in tf.items():
score = math.log(1 + cnt) * get_idf(term)
assert type(score) == float
score_d[term] = score
score_d_surface_form: Counter = Counter(
dict_key_map(lambda x: reverse_map[x], score_d))
ticker.tick()
yield score_d_surface_form
def run_random_walk_score_with_weight():
d_ids: List[int] = list(load_dev_claim_ids())
claims = get_claims_from_ids(d_ids)
top_k = 7
q_tf_replace = dict(load_from_pickle("random_walk_score_100"))
q_tf_replace = dict_key_map(lambda x: int(x), q_tf_replace)
bm25 = get_bm25_module()
pred = pc_predict_vector_query_and_reweight(bm25, q_tf_replace, claims,
top_k, {'k1': 0.5})
print(evaluate(pred))
def run_random_walk_score():
d_ids: List[int] = list(load_dev_claim_ids())
claims = get_claims_from_ids(d_ids)
top_k = 7
q_tf_replace = dict(load_from_pickle("random_walk_score_100"))
q_tf_replace = dict_key_map(lambda x: int(x), q_tf_replace)
#q_tf_replace = dict(load_from_pickle("pc_dev_par_tf"))
#q_tf_replace = dict(load_from_pickle("bias_random_walk_dev_plus_all"))
bm25 = get_bm25_module()
pred = pc_predict_from_vector_query(bm25, q_tf_replace, claims, top_k)
print(evaluate(pred))
def main():
config = load_run_config()
dvp_list = pickle.load(open(config['dvp_path'], "rb"))
dvp_to_correctness_dict = dvp_to_correctness(dvp_list, config)
def convert_key(key: Tuple[str, Tuple[str, int]]):
qid, (doc_id, doc_idx) = key
return qid, "{}_{}".format(doc_id, doc_idx)
save_dict = dict_key_map(convert_key, dvp_to_correctness_dict)
pickle.dump(save_dict, open(config['save_path'], "wb"))
def run_random_walk_score():
d_ids: List[int] = list(load_dev_claim_ids())
claims = get_claims_from_ids(d_ids)
top_k = 20
bm25 = get_bm25_module()
#df, N = get_idf()
#bm25.df = df
#bm25.N = N
q_tf_replace_0 = dict(load_from_pickle("random_walk_score_100"))
q_tf_replace = dict(load_from_pickle("dev_claim_random_walk_debug2"))
q_tf_replace = dict_key_map(lambda x: int(x), q_tf_replace)
pc_predict_to_inspect(bm25, q_tf_replace, q_tf_replace_0, claims, top_k)
def get_extended_eval_candidate_as_qck_raw(
split) -> Dict[str, List[QCKCandidate]]:
c: Dict[int, List[int]] = get_extended_eval_candidate(split)
def convert_candidates(candidates: List[int]) -> List[QCKCandidate]:
p_texts = lmap(perspective_getter, candidates)
l: List[QCKCandidate] = []
for pid, text in zip(candidates, p_texts):
l.append(QCKCandidate(str(pid), text))
return l
c2: Dict[int, List[QCKCandidate]] = dict_value_map(convert_candidates, c)
return dict_key_map(str, c2)
def make_qcknc_problem(
passage_score_path: FilePath,
info_path: FilePath,
config_path: FilePath,
split: str,
save_name: str,
) -> None:
candidate_dict: Dict[int, List[Dict]] = dict(
get_eval_candidates_from_pickle(split))
queries: List[QCKQuery] = get_qck_queries(split)
config = json.load(open(config_path, "r"))
def get_pids(l: List[Dict]) -> List[str]:
return lmap(lambda x: x['pid'], l)
candidate_id_dict_1: Dict[int, List[str]] = dict_value_map(
get_pids, candidate_dict)
candidate_id_dict: Dict[str,
List[str]] = dict_key_map(str, candidate_id_dict_1)
all_candidate_ids = set(flatten(candidate_id_dict.values()))
candidate_dict: Dict[str, QCKCandidate] = {
cid: get_qck_candidate_from_candidate_id(cid)
for cid in all_candidate_ids
}
data_id_to_info: Dict = load_combine_info_jsons(info_path, qk_convert_map)
print("number of dat info ", len(data_id_to_info))
qk_result: List[Tuple[str, List[QKOutEntry]]] = collect_good_passages(
data_id_to_info, passage_score_path, config)
query_dict = {q.query_id: q for q in queries}
payloads = qck_from_qk_results(qk_result, candidate_id_dict, query_dict,
candidate_dict)
out_dir = os.path.join(output_path, "cppnc")
exist_or_mkdir(out_dir)
save_path = os.path.join(out_dir, save_name + ".tfrecord")
data_id_man = write_qck_as_tfrecord(save_path, payloads)
info_save_path = os.path.join(out_dir, save_name + ".info")
print("Payload size : ", len(data_id_man.id_to_info))
json.dump(data_id_man.id_to_info, open(info_save_path, "w"))
print("tfrecord saved at :", save_path)
print("info saved at :", info_save_path)