def load_data(data_name): print("reading data") if data_name == "run_clueweb": path_list = clueweb_pair_prediction_list() data, data_len = combine_prediction_list(path_list) print("total of {} data".format(data_len)) pickle_path = get_clueweb_out_pickle_path() elif data_name == "hydroponics": path_list = hydroponics_prediction_path_list() data, data_len = combine_prediction_list(path_list) pickle_path = os.path.join(output_path, data_name) elif data_name == "weather": path_list = weather_prediction_path_list() data, data_len = combine_prediction_list(path_list) pickle_path = os.path.join(output_path, data_name) elif data_name == "first_abortion": data = EstimatorPredictionViewer(os.path.join(output_path, "nli_prediction")) print("total of {} data".format(data.data_len)) pickle_path = os.path.join(output_path, "abortion_nli_prediction_analysis") elif data_name == "rerun_abortion": data = EstimatorPredictionViewer(os.path.join(output_path, "abortion_contradiction")) print("total of {} data".format(data.data_len)) pickle_path = os.path.join(output_path, "abortion_nli_prediction_analysis2") else: data = EstimatorPredictionViewer(os.path.join(output_path, data_name)) print("total of {} data".format(data.data_len)) pickle_path = os.path.join(output_path, data_name + "_analysis") return data, pickle_path
def collect_data_w_cpid(prediction_file, info: Dict, logit_to_score) \ -> List[Dict]: data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) out = [] for entry in data: logits = entry.get_vector("logits") score = logit_to_score(logits) data_id = entry.get_vector("data_id")[0] confidence = get_confidence_or_rel_score(entry) try: cur_info = info[str(data_id)] cid = cur_info['cid'] pid = cur_info['pid'] cpid = CPIDPair((cid, pid)) cur_info['cpid'] = cpid cur_info['score'] = score cur_info['confidence'] = confidence out.append(cur_info) except KeyError as e: print("Key error") print("data_id", data_id) pass return out
def read_passage_scores(prediction_file, info: Dict, recover_subtokens ) \ -> Dict[int, List[Dict]] : data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) output: Dict[int, List] = defaultdict(list) fail_cnt =0 for entry in data: logits = entry.get_vector("logits") data_id = entry.get_vector("data_id")[0] try: cur_info = info[int(data_id)] cid = cur_info['cid'] d = { 'cid': cid, 'passage': recover_subtokens(entry.get_vector("input_ids")), 'logits': logits, 'data_id': data_id, } output[cid].append(d) except KeyError as e: print("Key error") print("data_id", data_id) fail_cnt += 1 if fail_cnt > 100: raise Exception() pass return output
def count_and_save(j): p = os.path.join(output_path, "clueweb12_13B_pair", "nli_prediction_{}".format(j)) d = EstimatorPredictionViewer(p) r = count_contradiction(d) p = os.path.join(output_path, "clueweb12_13B_pair_summary_{}".format(j)) pickle.dump(r, open(p, "wb"))
def combine_pc_rel_with_cpid(prediction_file, info: Dict) \ -> Dict[DataID, Tuple[CPIDPair, Logits, Logits]]: data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) out_d: Dict[DataID, Tuple[CPIDPair, Logits, Logits]] = {} num_append = 0 last_claim = None prev_data_id = None ticker = TimeEstimator(data.data_len) for entry in data: ticker.tick() logits = entry.get_vector("logits") data_id = entry.get_vector("data_id")[0] try: cur_info = info[data_id] if 'cid' in cur_info: cid = cur_info['cid'] last_claim = cid, logits prev_data_id = data_id elif 'pid' in cur_info: pid = cur_info['pid'] cid, c_logits = last_claim cpid = CPIDPair((cid, pid)) out_d[data_id] = (cpid, c_logits, logits) out_d[prev_data_id] = (cpid, c_logits, logits) num_append += 1 else: assert False except KeyError as e: print(e) pass return out_d
def save_ranked_list(prediction_path, meta_info, save_path): data = EstimatorPredictionViewer(prediction_path) q_dict = {} for entry in data: data_id = entry.get_vector('data_id')[0] scores = entry.get_vector('logits') q_id, doc_id = meta_info[data_id] if q_id not in q_dict: q_dict[q_id] = [] probs = softmax(scores) q_dict[q_id].append((doc_id, probs[1])) def add_rank( ranked_list: List[Tuple[str, float]]) -> List[Tuple[str, int, float]]: ranked_list.sort(key=lambda x: x[1], reverse=True) ranked_list = [(doc_id, rank, score) for rank, (doc_id, score) in enumerate(ranked_list)] return ranked_list q_dict_new = dict_value_map(add_rank, q_dict) write_ranked_list_from_d(q_dict_new, save_path)
def load_prediction(pred_path) -> List[Tuple[str, List[np.ndarray]]]: data = EstimatorPredictionViewer(pred_path) def parse_entry(entry) -> Tuple[str, np.ndarray]: input_tokens: Segment = entry.get_tokens('input_ids') logits = entry.get_vector("logits") probs = softmax(logits) key = input_tokens_to_key(input_tokens) return key, probs parsed_data: List[Tuple[str, np.ndarray]] = lmap(parse_entry, data) keys: List[str] = unique_from_sorted(left(parsed_data)) grouped: Dict[str, List[Tuple[str, np.ndarray]]] = group_by(parsed_data, lambda x: x[0]) def fetch_scores(key): l = [] for k2, score in grouped[key]: assert key == k2 l.append(score) return key, l results: List[Tuple[str, List[np.ndarray]]] = lmap(fetch_scores, keys) return results
def collect_scores(prediction_file, info: Dict, logit_to_score) \ -> Dict[DataID, Tuple[CPIDPair, float]]: data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) out_d: Dict[DataID, Tuple[CPIDPair, float]] = {} for entry in data: logits = entry.get_vector("logits") score = logit_to_score(logits) data_id = entry.get_vector("data_id")[0] try: cur_info = info[str(data_id)] if 'kdp' in cur_info: parse_info_inner(cur_info, qck_convert_map, True) cid = int(cur_info['query'].query_id) pid = int(cur_info['candidate'].id) elif 'query' in cur_info: parse_info_inner(cur_info, qc_convert_map, True) cid = int(cur_info['query'].query_id) pid = int(cur_info['candidate'].id) else: cid = cur_info['cid'] pid = cur_info['pid'] cpid = CPIDPair((cid, pid)) out_d[data_id] = (cpid, score) except KeyError as e: print("Key error", e) print("data_id", data_id) pass return out_d
def collect_pc_rel_score(prediction_file, info: Dict): data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) group_by_key = {} num_append = 0 last_claim = None ticker = TimeEstimator(data.data_len) for entry in data: ticker.tick() logits = entry.get_vector("logits") data_id = entry.get_vector("data_id")[0] try: cur_info = info[data_id] if 'cid' in cur_info: cid = cur_info['cid'] last_claim = cid, logits elif 'pid' in cur_info: pid = cur_info['pid'] cid, c_logits = last_claim key = cid, pid if key not in group_by_key: group_by_key[key] = [] group_by_key[key].append((c_logits, logits)) num_append += 1 else: assert False except KeyError as e: print(e) pass print(num_append) return group_by_key
def eval(file_name): data = EstimatorPredictionViewer(file_name) for entry in data: entry.get_vector("masked_label_ids_label") entry.get_vector("is_test_inst") print(entry.get_vector("is_test_inst"), entry.get_vector("masked_lm_example_loss_label"))
def combine_prediction_list(path_list): data_list = [] data_len = 0 for p in path_list: d = EstimatorPredictionViewer(p) data_len += d.data_len data_list.append(d) data = iter_fn(data_list) return data, data_len
def collect_by_order(input_file, feature_data: List[PerspectiveCandidate]): predictions = EstimatorPredictionViewer(input_file) print("prediction : {}".format(predictions.data_len)) print("feature_data : {}".format(len(feature_data))) score_d: Dict[CPID, float] = {} for pred_entry, pc_candidate in zip(predictions, feature_data): logits = pred_entry.get_vector("logits") probs = softmax(logits) score = probs[1] cpid = CPID("{}_{}".format(pc_candidate.cid, pc_candidate.pid)) score_d[cpid] = score return score_d
def pc_eval(pred_path: FilePath, label_path: FilePath, option="avg"): data = EstimatorPredictionViewer(pred_path) raw_predictions: List[Tuple[str, List[float]]] = load_prediction(data) keys, reduced_scores = reduce_score(raw_predictions, option) predictions = zip(keys, reduced_scores) labels_d: Dict[str, int] = load_label(label_path) label_list: List[int] = lmap(lambda x: labels_d[x], keys) pred = lmap(lambda x: int(x > 0.5), reduced_scores) print(reduced_scores) print(pred) num_correct = np.count_nonzero(np.equal(pred, label_list)) print("Acc : ", num_correct / len(label_list))
def load_prediction(pred_path) -> Dict[str, List[Tuple[str, float, Segment]]]: data = EstimatorPredictionViewer(pred_path) def parse_entry(entry) -> Tuple[str, float, Segment]: input_tokens: Segment = entry.get_tokens('input_ids') logits = entry.get_vector("logits") probs = softmax(logits) key = input_tokens_to_key(input_tokens) score = probs[1] return key, score, input_tokens parsed_data: List[Tuple[str, float, Segment]] = lmap(parse_entry, data) grouped: Dict[str, List[Tuple[str, float, Segment]]] = group_by(parsed_data, lambda x: x[0]) return grouped
def main(): data = EstimatorPredictionViewer(sys.argv[1]) rows = [] for e in data: label_ids = e.get_vector("label_ids") logits = e.get_vector("logits") probs = softmax(logits, -1) predict_label = 1 if probs[1] > 0.5 else 0 decision = "Y" if predict_label == label_ids else "N" row = [label_ids, probs[1], decision] rows.append(row) rows.sort(key=lambda x: x[1], reverse=True) print_table(rows)
def main(): prediction_file = sys.argv[1] pred_data = EstimatorPredictionViewer(prediction_file) def bin_fn(score): return str(int(score * 100)) bin = BinHistogram(bin_fn) for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e.get_vector('logits')) bin.add(score) print(bin.counter.keys()) for i in range(101): key = str(i) if key in bin.counter: print(key, bin.counter[key])
def main(): file_path = sys.argv[1] name = os.path.basename(file_path) viewer = EstimatorPredictionViewer(file_path) html = HtmlVisualizer("toke_score_gold.html") stopwords = load_stopwords_for_query() skip = 10 for entry_idx, entry in enumerate(viewer): if entry_idx % skip != 0: continue tokens = entry.get_tokens("input_ids") input_ids = entry.get_vector("input_ids") label_ids = entry.get_vector("label_ids") label_ids = np.reshape(label_ids, [-1, 2]) log_label_ids = np.log(label_ids + 1e-10) seg1, seg2 = split_p_h_with_input_ids(tokens, input_ids) pad_idx = tokens.index("[PAD]") assert pad_idx > 0 logits = entry.get_vector("logits") cells = [] cells2 = [] for idx in range(pad_idx): probs = label_ids[idx] token = tokens[idx] score = probs[0] color = "B" if score > 0 else "R" highlight_score = min(abs(score) * 10000, 100) if token in stopwords: highlight_score = 0 if token in seg1: highlight_score = 50 color = "G" c = Cell(token, highlight_score=highlight_score, target_color=color) cells.append(c) html.multirow_print_from_cells_list([cells, cells2]) if entry_idx > 10000: break
def aawd_pred_histogram(): prediction_file = at_output_dir("clue_counter_arg", "ada_argu3_aawd_20000.score") prediction_file = at_output_dir("clue_counter_arg", "ada_aawd5_clue.4000.score") pred_data = EstimatorPredictionViewer(prediction_file) def bin_fn(score): return str(int(score * 1000)) bin = BinHistogram(bin_fn) for idx, e in enumerate(pred_data): score = logit_to_score_softmax(e.get_vector('logits')) bin.add(score) for i in range(101): key = str(i) if key in bin.counter: print(key, bin.counter[key])
def collect_scores_and_confidence(prediction_file, info: Dict, logit_to_score) \ -> Dict[DataID, Tuple[CPIDPair, float, float]]: data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) out_d: Dict[DataID, Tuple[CPIDPair, float, float]] = {} for entry in data: logits = entry.get_vector("logits") score = logit_to_score(logits) data_id = entry.get_vector("data_id")[0] confidence = get_confidence_or_rel_score(entry) try: cur_info = info[str(data_id)] cid = cur_info['cid'] pid = cur_info['pid'] cpid = CPIDPair((cid, pid)) out_d[data_id] = (cpid, score, confidence) except KeyError as e: print("Key error") print("data_id", data_id) pass return out_d
def get_cpid_score(pred_path: FilePath, cpid_resolute_d: Dict[str, CPID], option="avg") -> Dict[CPID, float]: data = EstimatorPredictionViewer(pred_path) raw_predictions: List[Tuple[str, List[float]]] = load_prediction(data) keys, reduced_scores = reduce_score(raw_predictions, option) cpid_list = [] n_not_found = 0 for x in keys: try: cpid_list.append(cpid_resolute_d[x]) except KeyError as e: print("not found", x) n_not_found += 1 if n_not_found: print("{} missing from text -> cpid resolution".format(n_not_found)) # cpid_list = lmap(lambda x: cpid_resolute_d[x], keys) return dict(zip(cpid_list, reduced_scores))
def show(): p = os.path.join(output_path, "pair_eval_1") d = EstimatorPredictionViewer(p) pred0 = load_from_pickle("cont_model_0") labels = get_label_as_or() entries = [] entries2 = [] for idx, entry in enumerate(d): logits = entry.get_vector("logits") probs = softmax(logits) pred = np.argmax(probs) entries.append(probs[2]) entries2.append(random.random()) #print(idx, pred==labels[idx], pred, labels[idx], probs) labels_binary = list([t == 2 for t in labels]) print(roc_auc_score(labels, entries)) print(roc_auc_score(labels, entries2)) print(roc_auc_score(labels, pred0))
def collect_info(prediction_file, info: Dict, logit_to_score) -> Dict[CPIDPair, List[Tuple[float, float, Dict]]]: data = EstimatorPredictionViewer(prediction_file) print("Num data ", data.data_len) out_d: Dict[CPIDPair, List[Tuple[float, float, Dict]]] = defaultdict(list) for entry in data: logits = entry.get_vector("logits") score = logit_to_score(logits) rel_score = entry.get_vector("rel_score")[0] data_id = entry.get_vector("data_id")[0] try: cur_info = info[str(data_id)] cid = cur_info['cid'] pid = cur_info['pid'] cpid = CPIDPair((cid, pid)) out_d[cpid].append((score, rel_score, cur_info)) except KeyError as e: print("Key error") print("data_id", data_id) pass return out_d
def collect_pipeline2_score(prediction_path, pc_rel_info) -> Dict[CPID, List[float]]: info_d = load_from_pickle(pc_rel_info) print('info_d', len(info_d)) def get_cpid(data_id, info_d) -> CPID: try: info_1 = info_d[data_id - 1] info_2 = info_d[data_id] cid = info_1['cid'] pid = info_2['pid'] except KeyError: info_1 = info_d[data_id] info_2 = info_d[data_id + 1] cid = info_1['cid'] pid = info_2['pid'] return CPID("{}_{}".format(cid, pid)) data = EstimatorPredictionViewer(prediction_path) print("Num data ", data.data_len) ticker = TimeEstimator(data.data_len) score_list_d: Dict[CPID, List] = {} for entry in data: ticker.tick() logits = entry.get_vector("logits") probs = softmax(logits) score = probs[1] data_id = entry.get_vector("data_id")[0] cpid: CPID = get_cpid(data_id, info_d) if cpid not in score_list_d: score_list_d[cpid] = [] score_list_d[cpid].append(score) return score_list_d
def main(pred_path, info_path, output_path): pred_data = EstimatorPredictionViewer(pred_path) info = load_from_pickle(info_path) prediction_to_ranked_list(pred_data, info, output_path)
def dir_itr(dir_path): for file_path in get_dir_files(dir_path): data = EstimatorPredictionViewer(file_path) for entry in data: yield entry
def load_preditions(path): data = EstimatorPredictionViewer(path) for entry in data: logits = entry.get_vector("logits") yield logits
def load_preditions(path): data = EstimatorPredictionViewer(path) for entry in data: input_ids = entry.get_vector("input_ids") logits = entry.get_vector("logits") yield input_ids, logits