def extract_qk_unit(info_path, pred_path, config_path) -> Iterable[QKUnit]: info = load_combine_info_jsons(info_path, qk_convert_map, False) predictions = join_prediction_with_info(pred_path, info) grouped: Dict[str, List[Dict]] = group_by(predictions, lambda x: x['query'].query_id) config = json.load(open(config_path, "r")) score_cut = config['score_cut'] top_k = config['top_k'] def is_good(entry): return get_regression_score(entry) > score_cut select_rate_list = [] qk_units = [] for qid, entries in grouped.items(): any_entry = entries[0] query = any_entry['query'] good_entries = lfilter(is_good, entries) good_entries.sort(key=get_regression_score, reverse=True) selected_entries = good_entries[:top_k] if not selected_entries: continue kd_list = lmap(lambda x: x['kdp'], selected_entries) qk_units.append((query, kd_list)) select_rate = len(selected_entries) / len(entries) select_rate_list.append(select_rate) print("{} of {} qk units selected".format(len(qk_units), len(grouped))) print("average select rate", average(select_rate_list)) return qk_units
def summarize_score(info: Dict, prediction_file_path: str, f_handler: FormatHandler, score_type) -> Iterable[TrecRankedListEntry]: key_logit = "logits" data: List[Dict] = join_prediction_with_info(prediction_file_path, info, ["data_id", key_logit]) def logit_to_score_softmax(logit): return scipy.special.softmax(logit)[1] def get_score(entry): if score_type == "softmax": return logit_to_score_softmax(entry['logits']) elif score_type == "raw": return entry[key_logit][0] elif score_type == "scalar": return entry[key_logit] elif score_type == "tuple": return entry[key_logit][1] else: assert False grouped: Dict[Tuple[str, str], List[Dict]] = group_by(data, f_handler.get_pair_id) print("Group size:", len(grouped)) out_d = {} for pair_id, items in grouped.items(): scores = lmap(get_score, items) query_id, doc_id = pair_id out_d[pair_id] = scores for score in scores: yield TrecRankedListEntry(query_id, doc_id, 0, score, "")
def main(): info_dir = os.path.join(job_man_dir, "best_seg_prediction_gen_train_info") job_id = 0 info_file_path = os.path.join(info_dir, str(job_id) + ".info") print(info_file_path) info = json.load(open(info_file_path, "r")) prediction_dir = "output/mmd_ss/mmd_Z_50000" prediction_file = os.path.join(prediction_dir, str(job_id) + ".score") pred_data: List[Dict] = join_prediction_with_info(prediction_file, info) target_qdid = ("1000633", "D144400") saved_entries = [] for key, entry in info.items(): if entry['qid'] == "1000633" and entry['doc_id'] == 'D144400': saved_entries.append(entry) print(entry) print('--') for entry in pred_data: if entry['qid'] == "1000633" and entry['doc_id'] == 'D144400': print(entry) qid = "1000633" sr_path = os.path.join(job_man_dir, "seg_resource_train", qid) sr_per_query: SRPerQuery = load_pickle_from(sr_path) for sr_per_query_doc in sr_per_query.sr_per_query_doc: if sr_per_query_doc.doc_id == "D144400": print("doc {} has {} segs".format(sr_per_query_doc.doc_id, len(sr_per_query_doc.segs)))
def summarize_score_wo_merge(info: Dict, prediction_file_path: str, f_handler: FormatHandler, score_type) -> Dict[Tuple[str, str], float]: key_logit = "logits" data: List[Dict] = join_prediction_with_info(prediction_file_path, info, ["data_id", key_logit]) def logit_to_score_softmax(logit): return scipy.special.softmax(logit)[1] def get_score(entry): if score_type == "softmax": return logit_to_score_softmax(entry['logits']) elif score_type == "raw": return entry[key_logit][0] elif score_type == "scalar": return entry[key_logit] elif score_type == "tuple": return entry[key_logit][1] else: assert False grouped: Dict[Tuple[str, str], List[Dict]] = group_by(data, f_handler.get_pair_id) tprint("Group size:", len(grouped)) out_d = {} for pair_id, items in grouped.items(): query_id, doc_id = pair_id scores = lmap(get_score, items) for idx, score in enumerate(scores): new_doc_id = "{}_{}".format(doc_id, idx) out_d[(query_id, new_doc_id)] = score return out_d
def summarize_score(info: Dict, prediction_file_path: str, f_handler: FormatHandler, combine_score: Callable, score_type) -> Dict[Tuple[str, str], float]: key_logit = "logits" data: List[Dict] = join_prediction_with_info(prediction_file_path, info, ["data_id", key_logit]) def logit_to_score_softmax(logit): return scipy.special.softmax(logit)[1] def get_score(entry): if score_type == "softmax": return logit_to_score_softmax(entry['logits']) elif score_type == "raw": return entry[key_logit][0] elif score_type == "scalar": return entry[key_logit] elif score_type == "tuple": return entry[key_logit][1] else: assert False grouped: Dict[Tuple[str, str], List[Dict]] = group_by(data, f_handler.get_pair_id) tprint("Group size:", len(grouped)) out_d = {} for pair_id, items in grouped.items(): scores = lmap(get_score, items) final_score = combine_score(scores) out_d[pair_id] = final_score num_items_per_group = average(lmap(len, grouped.values())) tprint("Num items per group : ", num_items_per_group) return out_d
def load_baseline_score_d(baseline_info_file_path, pred_path, is_info_from_pickle) -> Dict[Tuple[str, str], float] : info = load_combine_info_jsons(baseline_info_file_path, qck_convert_map) predictions: List[Dict] = join_prediction_with_info(pred_path, info, ["logits"], is_info_from_pickle) out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, predictions) baseline_d: Dict[Tuple[str, str], float] = {} for e in out_entries: key = e.query.query_id, e.candidate.id score = logit_to_score_softmax(e.logits) baseline_d[key] = score return baseline_d
def load_qk_score(config) -> List[QKOutEntry]: info_path = config['info_path'] passage_score_path = config['pred_path'] score_type = config['score_type'] fetch_field_list = ["logits", "input_ids", "data_id"] data_id_to_info: Dict = load_combine_info_jsons(info_path, qk_convert_map) data: List[Dict] = join_prediction_with_info(passage_score_path, data_id_to_info, fetch_field_list) qk_out_entries: List[QKOutEntry] = lmap(QKOutEntry.from_dict2, data) return qk_out_entries
def calculate_score(info, pred_path, baseline_score: Dict[Tuple[str, str], float], str_data_id=False) -> List[DocValueParts]: predictions: List[Dict] = join_prediction_with_info( pred_path, info, ["logits"], str_data_id) out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, predictions) labels: Dict[str, List[str]] = load_labels() doc_score_parts: List[DocValueParts] = get_doc_value_parts( out_entries, baseline_score, labels) return doc_score_parts
def show_high(): info_save_path = at_output_dir("clue_counter_arg", "clue_f5.tfrecord.info") info = json.load(open(info_save_path, "r")) # prediction_file = at_output_dir("clue_counter_arg", "ada_aawd4_clue.4000.score") prediction_file = at_output_dir("clue_counter_arg", "ada_aawd5_clue.4000.score") pred_data = join_prediction_with_info(prediction_file, info) for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e['logits']) if int(score * 100) == 13: print(e['text'])
def summarize_score(info: Dict, prediction_file_path: str, score_type) -> Dict[Tuple[str, str, int], float]: key_logit = "logits" data: List[Dict] = join_prediction_with_info(prediction_file_path, info, ["data_id", key_logit]) score_d: Dict[Tuple[str, str, int], float] = {} for entry in data: score = get_score_from_logit(score_type, entry['logits']) key = entry['query_id'], entry['doc_id'], entry['passage_idx'] score_d[key] = score return score_d
def doc_score_predictions(): passage_score_path = "output/cppnc/qknc_val" info = load_combine_info_jsons("output/cppnc/qknc_val.info", qk_convert_map) data = join_prediction_with_info(passage_score_path, info) grouped: Dict[str, List[Dict]] = group_by(data, lambda x: x['query'].query_id) def get_score_from_logit(logits): return scipy.special.softmax(logits)[1] for cid, passages in grouped.items(): scores: List[float] = lmap(lambda d: get_score_from_logit(d['logits']), passages) yield cid, scores
def main(config): info_dir = config['info_path'] prediction_file = config['pred_path'] f_handler = get_format_handler("qck") info = load_combine_info_jsons(info_dir, f_handler.get_mapping(), f_handler.drop_kdp()) data: List[Dict] = join_prediction_with_info(prediction_file, info, ["data_id", "logits"]) out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, data) qrel: Dict[str, Dict[str, int]] = load_qrels_structured(config['qrel_path']) def get_label(query_id, candi_id): if candi_id in qrel[query_id]: return qrel[query_id][candi_id] else: return 0 def logit_to_score_softmax(logit): return scipy.special.softmax(logit)[1] grouped: Dict[str, List[QCKOutEntry]] = group_by(out_entries, lambda x: x.query.query_id) for query_id, items in grouped.items(): raw_kdp_list = [(x.kdp.doc_id, x.kdp.passage_idx) for x in items] kdp_list = unique_list(raw_kdp_list) raw_candi_id_list = [x.candidate.id for x in items] candi_id_list = unique_list(raw_candi_id_list) logit_d = {(x.candidate.id, (x.kdp.doc_id, x.kdp.passage_idx)): x.logits for x in items} labels = [get_label(query_id, candi_id) for candi_id in candi_id_list] head_row0 = [" "] + labels head_row1 = [" "] + candi_id_list rows = [head_row0, head_row1] for kdp_sig in kdp_list: row = [kdp_sig] for candi_id in candi_id_list: try: score = logit_to_score_softmax(logit_d[candi_id, kdp_sig]) score_str = "{0:.2f}".format(score) except KeyError: score_str = "-" row.append(score_str) rows.append(row) print(query_id) print_table(rows)
def main(config): info_path = sys.argv[1] pred_path = sys.argv[2] info = load_combine_info_jsons(info_path, True) predictions = join_prediction_with_info(pred_path, info, silent=True) out_entries: List[OutEntry] = lmap(OutEntry.from_dict, predictions) g = group_by(out_entries, lambda x: x.doc_id) for doc_id in g: entries: List[OutEntry] = g[doc_id] scores = list([logit_to_score_softmax(e.logits) for e in entries]) print(doc_id, max(scores))
def enum_best_segments_always(pred_path, info) -> Iterable[Dict]: entries = join_prediction_with_info(pred_path, info) grouped = group_by(entries, lambda e: (e['query_id'], e['doc_id'])) for key in grouped: sub_entries = grouped[key] def get_score(e): return logit_to_score_softmax(e['logits']) max_idx = find_max_idx(sub_entries, get_score) selected_raw_entry = sub_entries[max_idx] yield selected_raw_entry
def main(): baseline_score: Dict[Tuple[str, str], float] = load_baseline() score_save_path = sys.argv[1] info_path = sys.argv[2] info = load_combine_info_jsons(info_path, qck_convert_map, False) # calculate score for each kdp predictions: List[Dict] = join_prediction_with_info( score_save_path, info, ["logits"], True) out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, predictions) labels: Dict[str, List[str]] = load_labels() doc_score_parts: List[DocValueParts2] = get_doc_value_parts2( out_entries, baseline_score, labels) summary_save_path = sys.argv[3] pickle.dump(doc_score_parts, open(summary_save_path, "wb"))
def get_f5_tids_score_d_from_bert(): info_save_path = at_output_dir("clue_counter_arg", "clue_f5.tfrecord.info") info = json.load(open(info_save_path, "r")) prediction_file = at_output_dir("clue_counter_arg", "ada_aawd5_clue.4000.score") pred_data = join_prediction_with_info(prediction_file, info) score_d = {} for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e['logits']) text = e['text'] score_d[text] = score return score_d
def collect_and_save_score(config): info_path = config['info_path'] pred_path = config['pred_path'] save_path = config['save_path'] info = load_combine_info_jsons(info_path, qk_convert_map, False) predictions: List[Dict] = join_prediction_with_info(pred_path, info, ['data_id', 'logits', 'input_ids', 'label_ids'], ) outputs: Iterable[QKTokenLevelOutEntry] = map(QKTokenLevelOutEntry.from_dict, predictions) per_query_infos: Dict[str, Dict[WordAsID, np.array]] = group_average_per_query(outputs) pickle.dump(per_query_infos, open(save_path, "wb"))
def main(pred_file_path: str, info_file_path: str, info_file_path2: str, save_name: str, input_type: str, qrel_path: str, ): judgement = load_qrels_structured(qrel_path) def get_label(key): query_id, doc_id = key try: return judgement[query_id][doc_id] except KeyError: return 0 f_handler = get_format_handler(input_type) info: Dict = load_combine_info_jsons(info_file_path, f_handler.get_mapping(), f_handler.drop_kdp()) info2: Dict = load_combine_info_jsons(info_file_path2, f_handler.get_mapping(), f_handler.drop_kdp()) doc_length = get_doc_length_info(info2) key_logit = "logits" data: List[Dict] = join_prediction_with_info(pred_file_path, info, ["data_id", key_logit]) grouped = group_by(data, f_handler.get_pair_id) cnt = Counter() for key, entries in grouped.items(): if not get_label(key): continue seg_groups = {} for e in entries: probs = scipy.special.softmax(e['logits'])[:, 1] seg_groups[e['idx']] = probs indices = list(seg_groups.keys()) indices.sort() assert max(indices) == len(indices) - 1 all_probs = [] for seg_group_idx in seg_groups.keys(): all_probs.extend(seg_groups[seg_group_idx]) num_seg = doc_length[key] max_idx = np.argmax(all_probs[:num_seg]) cnt[(max_idx, num_seg)] += 1 save_to_pickle(cnt, save_name)
def main(config): info = load_combine_info_jsons(config['info_path']) predictions: List[Dict] = join_prediction_with_info(config['pred_path'], info, ["data_ids", "logits"], True, "data_ids") entries: List[OutEntry] = lmap(OutEntry.from_dict, predictions) def is_pos(e: OutEntry): return logit_to_score_softmax(e.logits) > 0.5 pos_entries = filter(is_pos, entries) rows = [] for e in pos_entries: row = [e.cid, e.pid, e.doc_id, e.sent_idx] rows.append(row) print_table(rows)
def load_scores(info_file_path, prediction_file_path): input_type = "qc" f_handler = get_format_handler(input_type) tprint("Loading json info") info: Dict = load_combine_info_jsons(info_file_path, f_handler.get_mapping(), f_handler.drop_kdp()) key_logit = "logits" tprint("Reading predictions...") data: List[Dict] = join_prediction_with_info(prediction_file_path, info, ["data_id", key_logit]) grouped: Dict[Tuple[str, str], List[Dict]] = group_by(data, f_handler.get_pair_id) print("number of groups:", len(grouped)) return grouped
def extract_qk_unit(info_path, pred_path): info = load_combine_info_jsons(info_path, qk_convert_map, False) predictions = join_prediction_with_info(pred_path, info) grouped: Dict[str, List[Dict]] = group_by(predictions, lambda x: x['query'].query_id) rows = [] for qid, entries in grouped.items(): any_entry = entries[0] query = any_entry['query'] rows.append([query.query_id, query.text]) for entry in entries: row = [get_regression_score(entry), entry['kdp'].doc_id, entry['kdp'].passage_idx] rows.append(row) print_table(rows)
def main(config): info = load_combine_info_jsons(config['info_path']) predictions: List[Dict] = join_prediction_with_info( config['pred_path'], info) entries = lmap(OutEntry.from_dict, predictions) def get_doc_id(e: OutEntry): return e.doc_id grouped = group_by(entries, get_doc_id) for doc_id in grouped: doc_entries = grouped[doc_id] doc_entries.sort(key=lambda x: x.sent_idx) n_pos = 0 for s in doc_entries: if s.logits[1] > 0.5: n_pos += 1 tab_print(doc_id, n_pos, len(doc_entries))
def collect_good_passages(data_id_to_info: Dict[str, Dict], passage_score_path: FilePath, config: Dict ) -> List[Tuple[str, List[QKOutEntry]]]: global recover_subtokens recover_subtokens = get_recover_subtokens() score_cut = config['score_cut'] top_k = config['top_k'] score_type = config['score_type'] fetch_field_list = ["logits", "input_ids", "data_id"] data: List[Dict] = join_prediction_with_info(passage_score_path, data_id_to_info, fetch_field_list ) qk_out_entries: List[QKOutEntry] = lmap(QKOutEntry.from_dict, data) grouped: Dict[str, List[QKOutEntry]] = group_by(qk_out_entries, lambda x: x.query.query_id) def get_score_from_logit_local(logits) -> float: return get_score_from_logit(score_type, logits) def get_score(entry: QKOutEntry): return get_score_from_logit_local(entry.logits) def is_good(qk_out_entry: QKOutEntry): score = get_score_from_logit_local(qk_out_entry.logits) return score >= score_cut output = [] num_passges = [] for cid, passages in grouped.items(): good_passages = lfilter(is_good, passages) good_passages.sort(key=get_score, reverse=True) num_passges.append(len(good_passages)) if good_passages: output.append((cid, good_passages[:top_k])) else: scores = lmap(get_score, passages) scores.sort(reverse=True) print(num_passges) print("{} of {} query has passages".format(len(output), len(grouped))) return output
def main(): info_save_path = at_output_dir("clue_counter_arg", "clue_f5.tfrecord.info") info = json.load(open(info_save_path, "r")) prediction_file = at_output_dir("clue_counter_arg", "ada_aawd4_clue.4000.score") pred_data = join_prediction_with_info(prediction_file, info) def bin_fn(score): return str(int(score * 100)) bin = BinHistogram(bin_fn) for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e['logits']) bin.add(score) for i in range(101): key = str(i) if key in bin.counter: print(key, bin.counter[key])
def get_best_seg_info(self, job_id) -> Dict[Tuple[str, str], int]: info = json.load( open(os.path.join(self.info_dir, str(job_id) + ".info"), "r")) prediction_file = os.path.join(self.prediction_dir, str(job_id) + ".score") pred_data: List[Dict] = join_prediction_with_info( prediction_file, info) def get_score(entry): return self.logits_to_score(entry['logits']) qdid_grouped = group_by(pred_data, lambda d: (d['qid'], d['doc_id'])) qdid_to_max_seg_idx: Dict[Tuple[str, str], int] = {} for qdi, entries in qdid_grouped.items(): query_id, doc_id = qdi max_seg_idx = entries[find_max_idx(entries, get_score)]['seg_idx'] qdid_to_max_seg_idx[query_id, doc_id] = max_seg_idx return qdid_to_max_seg_idx
def summarize_score(info_dir, prediction_file) -> Dict[Tuple[str, str], float]: info = load_combine_info_jsons(info_dir, qckl_convert_map, False) print("Info has {} entries".format(len(info))) data: List[Dict] = join_prediction_with_info(prediction_file, info, ["data_id", "logits"]) def get_score(entry): return entry['logits'] grouped: Dict[Tuple[str, str], List[Dict]] = group_by(data, get_qc_pair_id) print("Group size:", len(grouped)) out_d = {} for pair_id, items in grouped.items(): scores = lmap(get_score, items) assert len(scores) == 1 final_score = scores[0] out_d[pair_id] = final_score num_items_per_group = average(lmap(len, grouped.values())) print("Num items per group : ", num_items_per_group) return out_d
def print_top_k(): k = 30 info_save_path = at_output_dir("clue_counter_arg", "clue_f5.tfrecord.info") info = json.load(open(info_save_path, "r")) prediction_file = at_output_dir("clue_counter_arg", "ada_aawd5_clue.4000.score") pred_data = join_prediction_with_info(prediction_file, info) simple_data = [] text_set = set() for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e['logits']) text = e['text'] if text in text_set: continue text_set.add(text) simple_data.append((text, score)) simple_data.sort(key=get_second, reverse=True) for text, score in simple_data[:k]: tab_print(score * 100, text)
def load_cppnc_score(fetch_field_list=None) -> Dict[str, Dict[str, List[Dict]]]: save_name = "qcknc_dense_val" score_name_list = [] for i in range(0, 17): score_name_list.append("qcknc_dense_val_{}".format(i)) out_dir = os.path.join(output_path, "cppnc") exist_or_mkdir(out_dir) info_file_path = os.path.join(out_dir, save_name + ".info") print("loading json info BEGIN") info = load_combine_info_jsons(info_file_path, qck_convert_map) print("loading json info DONE") all_predictions = [] for score_name in score_name_list: pred_file_path = os.path.join(out_dir, score_name + ".score") print(score_name) predictions = join_prediction_with_info(pred_file_path, info, fetch_field_list) all_predictions.extend(predictions) qid_grouped = group_by_qid_cid(all_predictions) return qid_grouped
def main(): config = load_run_config() info = load_combine_info_jsons(config['info_path'], qck_convert_map, False) label_d: Dict[int, List[int]] = get_claim_perspective_id_dict2() print("Info length:", len(info)) predictions: List[Dict] = join_prediction_with_info( config['pred_path'], info) print("Prediction length:", len(predictions)) out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, predictions) out_entries = out_entries[:10000] out_entries.sort(key=lambda x: logit_to_score_softmax(x.logits), reverse=True) def get_label(entry: QCKOutEntry): return int(entry.candidate.id) in label_d[int(entry.query.query_id)] rows = [] for entry in out_entries[:100]: label = get_label(entry) score = logit_to_score_softmax(entry.logits) print_info(entry, rows, score, label) print_table(rows)
def do_job(input_dir, output_dir, info_dir, label_info_path, max_entries, job_id): exist_or_mkdir(output_dir) info_output_dir = output_dir + "_info" exist_or_mkdir(info_output_dir) label_info: List[Tuple[str, str, int]] = json.load(open(label_info_path, "r")) label_info_d = {(str(a), str(b)): c for a, b, c in label_info} pred_path = os.path.join(input_dir, str(job_id) + ".score") #info_path = os.path.join(info_dir, str(job_id) + ".info") info_path = info_dir output_path = os.path.join(output_dir, str(job_id)) info_output_path = os.path.join(info_output_dir, str(job_id)) info = load_combine_info_jsons(info_path, qck_convert_map, True) fetch_field_list = ["vector", "data_id"] predictions = join_prediction_with_info(pred_path, info, fetch_field_list) def get_qid(entry): return entry['query'].query_id def get_candidate_id(entry): return entry['candidate'].id def pair_id(entry) -> Tuple[str, str]: return get_qid(entry), get_candidate_id(entry) groups: Dict[Tuple[str, str], List[Dict]] = group_by(predictions, pair_id) def get_new_entry(entries: List[Dict]): if not entries: return None vectors: Vectors = list([e['vector'] for e in entries]) key = pair_id(entries[0]) if key in label_info_d: label: Label = label_info_d[key] else: label: Label = 0 return vectors, label g2: Dict[Tuple[str, str], Tuple[Vectors, Label]] = dict_value_map(get_new_entry, groups) base = 100 * 1000 * job_id max_count = 100 * 1000 * (job_id + 1) data_id_manager = DataIDManager(base, max_count) def get_out_itr() -> Iterable[Tuple[int, Tuple[Vectors, Label]]]: for key, data in g2.items(): qid, cid = key data_info = { 'qid': qid, 'cid': cid, } data_id = data_id_manager.assign(data_info) yield data_id, data write_to_file(output_path, get_out_itr(), max_entries) json.dump(data_id_manager.id_to_info, open(info_output_path, "w"))