def draw2(in_file, out_file): filename = os.path.join(output_path, in_file) data = EstimatorPredictionViewerGosford(filename) html_writer = HtmlVisualizer(out_file, dark_mode=False) tokenizer = get_tokenizer() for inst_i, entry in enumerate(data): if inst_i > 100: break tokens = entry.get_tokens("input_ids") # tokens = entry.get_tokens("input_ids") prob1 = entry.get_vector("prob1") prob2 = entry.get_vector("prob2") real_loss1 = entry.get_vector("per_example_loss1") real_loss2 = entry.get_vector("per_example_loss2") masked_lm_positions = entry.get_vector("masked_lm_positions") for i, loc in enumerate(masked_lm_positions): tokens[loc] = "[{}:{}]".format(i, tokens[loc]) html_writer.multirow_print(data.cells_from_tokens(tokens)) row2 = [Cell("prob1:")] + data.cells_from_anything(prob1) row3 = [Cell("prob2:")] + data.cells_from_anything(prob2) row4 = [Cell("real_loss1:")] + data.cells_from_anything(real_loss1) row5 = [Cell("real_loss2:")] + data.cells_from_anything(real_loss2) html_writer.multirow_print_from_cells_list([row2, row3, row4, row5])
def view_grad_overlap_hidden(): filename = "ukp_feature_overlap.pickle" obj = pickle.load(open(os.path.join(output_path, filename), "rb")) out_name = filename.split(".")[0] + ".html" html_writer = HtmlVisualizer(out_name, dark_mode=False) data = EstimatorPredictionViewerGosford(filename) for inst_i, entry in enumerate(data): tokens = entry.get_mask_resolved_input_mask_with_input() h_overlap = entry.get_vector('h_overlap') std = np.std(h_overlap, axis=2) mean = np.mean(h_overlap, axis=2) h_overlap = np.sum(h_overlap, axis=2) highlight = lmap(is_mask, tokens) cells = data.cells_from_tokens(tokens, highlight) rows = [cells] for layer_i in range(12): e = h_overlap[layer_i, :] e = [v * 1e6 for v in e] cells = data.cells_from_scores(e) rows.append(cells) e = [v * 1e8 for v in std[layer_i, :]] cells2 = data.cells_from_scores(e) rows.append(cells2) print(entry.get_vector("masked_lm_example_loss")) html_writer.multirow_print_from_cells_list(rows, 40)
def main(): file_path = sys.argv[1] name = os.path.basename(file_path) viewer = EstimatorPredictionViewer(file_path) html = HtmlVisualizer("toke_score_gold.html") stopwords = load_stopwords_for_query() skip = 10 for entry_idx, entry in enumerate(viewer): if entry_idx % skip != 0: continue tokens = entry.get_tokens("input_ids") input_ids = entry.get_vector("input_ids") label_ids = entry.get_vector("label_ids") label_ids = np.reshape(label_ids, [-1, 2]) log_label_ids = np.log(label_ids + 1e-10) seg1, seg2 = split_p_h_with_input_ids(tokens, input_ids) pad_idx = tokens.index("[PAD]") assert pad_idx > 0 logits = entry.get_vector("logits") cells = [] cells2 = [] for idx in range(pad_idx): probs = label_ids[idx] token = tokens[idx] score = probs[0] color = "B" if score > 0 else "R" highlight_score = min(abs(score) * 10000, 100) if token in stopwords: highlight_score = 0 if token in seg1: highlight_score = 50 color = "G" c = Cell(token, highlight_score=highlight_score, target_color=color) cells.append(c) html.multirow_print_from_cells_list([cells, cells2]) if entry_idx > 10000: break
def show(filename): data = EstimatorPredictionViewerGosford(filename) html_writer = HtmlVisualizer("token_scoring.html", dark_mode=False) correctness = [] for entry in data: tokens = entry.get_tokens("input_ids") logits = entry.get_vector("logits") masks = entry.get_vector("label_masks") ids = entry.get_vector("labels") token_row = [] pred_row = [] gold_row = [] rows = [token_row, pred_row, gold_row] for idx, token in enumerate(tokens): token_cell = Cell(token) if token == "[PAD]": break model_score = logits[idx][0] if masks[idx]: correct = (model_score > 0 and ids[idx] > 0) or (model_score < 0 and ids[idx] < 0) color = "B" if correct else "R" if correct and (model_score > 0 and ids[idx] > 0) : color = "G" pred_cell = Cell("{0:.2f}".format(model_score), 100, target_color=color) gold_cell = Cell("{0:.2f}".format(ids[idx]), 100, target_color=color) else: token_cell = Cell(token) pred_cell = Cell("") gold_cell = Cell("") token_row.append(token_cell) pred_row.append(pred_cell) gold_row.append(gold_cell) html_writer.multirow_print_from_cells_list(rows, 20)
def main(): print("Loading scores...") cid_grouped: Dict[str, Dict[str, List[Dict]]] = load_cppnc_score_wrap() baseline_cid_grouped = load_baseline("train_baseline") gold = get_claim_perspective_id_dict() tokenizer = get_tokenizer() claim_d = load_train_claim_d() print("Start analyzing") html = HtmlVisualizer("cppnc_value_per_token_score.html") claim_cnt = 0 for cid, pid_entries_d in cid_grouped.items(): pid_entries_d: Dict[str, List[Dict]] = pid_entries_d pid_entries: List[Tuple[str, List[Dict]]] = list(pid_entries_d.items()) baseline_pid_entries = baseline_cid_grouped[int(cid)] baseline_score_d = fetch_score_per_pid(baseline_pid_entries) gold_pids = gold[int(cid)] ret = collect_score_per_doc(baseline_score_d, get_score_from_entry, gold_pids, pid_entries) passage_tokens_d = collect_passage_tokens(pid_entries) doc_info_d: Dict[int, Tuple[str, int]] = ret[0] doc_value_arr: List[List[float]] = ret[1] kdp_result_grouped = defaultdict(list) for doc_idx, doc_values in enumerate(doc_value_arr): doc_id, passage_idx = doc_info_d[doc_idx] avg_score = average(doc_values) kdp_result = doc_id, passage_idx, avg_score kdp_result_grouped[doc_id].append(kdp_result) s = "{} : {}".format(cid, claim_d[int(cid)]) html.write_headline(s) claim_cnt += 1 if claim_cnt > 10: break scores: List[float] = list([r[2] for r in doc_value_arr]) foreach(html.write_paragraph, lmap(str, scores)) for doc_id, kdp_result_list in kdp_result_grouped.items(): html.write_headline(doc_id) tokens, per_token_score = combine_collect_score(tokenizer, doc_id, passage_tokens_d, kdp_result_list) str_tokens = tokenizer.convert_ids_to_tokens(tokens) row = cells_from_tokens(str_tokens) for idx in range(len(str_tokens)): score = per_token_score[idx][0] norm_score = min(abs(score) * 10000, 100) color = "B" if score > 0 else "R" row[idx].highlight_score = norm_score row[idx].target_color = color rows = [row] nth = 0 any_score_found = True while any_score_found: any_score_found = False score_list = [] for idx in range(len(str_tokens)): if nth < len(per_token_score[idx]): score = per_token_score[idx][nth] any_score_found = True else: score = "-" score_list.append(score) def get_cell(score): if score == "-": return Cell("-") else: # 0.01 -> 100 norm_score = min(abs(score) * 10000, 100) color = "B" if score > 0 else "R" return Cell("", highlight_score=norm_score, target_color=color) nth += 1 if any_score_found: row = lmap(get_cell, score_list) rows.append(row) html.multirow_print_from_cells_list(rows)
def show_all(run_name, data_id): num_tags = 3 num_select = 1 pickle_name = "save_view_{}_{}".format(run_name, data_id) tokenizer = get_tokenizer() data_loader = get_modified_data_loader2(HPSENLI3(), NLIExTrainConfig()) explain_entries = load_from_pickle(pickle_name) explain_entries = explain_entries visualizer = HtmlVisualizer(pickle_name + ".html") tex_visulizer = TexTableNLIVisualizer(pickle_name + ".tex") tex_visulizer.begin_table() selected_instances = [[], [], []] for idx, entry in enumerate(explain_entries): x0, logits, scores = entry pred = np.argmax(logits) input_ids = x0 p, h = data_loader.split_p_h_with_input_ids(input_ids, input_ids) p_tokens = tokenizer.convert_ids_to_tokens(p) h_tokens = tokenizer.convert_ids_to_tokens(h) p_rows = [] h_rows = [] p_rows.append(cells_from_tokens(p_tokens)) h_rows.append(cells_from_tokens(h_tokens)) p_score_list = [] h_score_list = [] for j in range(num_tags): tag_name = data_generator.NLI.nli_info.tags[j] p_score, h_score = data_loader.split_p_h_with_input_ids(scores[j], input_ids) normalize_fn = normalize add = True if pred == "0": add = tag_name == "match" if pred == "1": add = tag_name == "mismatch" if pred == "2": add = tag_name == "conflict" def format_scores(raw_scores): def format_float(s): return "{0:.2f}".format(s) norm_scores = normalize_fn(raw_scores) cells = [Cell(format_float(s1), s2, False, False) for s1, s2 in zip(raw_scores, norm_scores)] if tag_name == "mismatch": set_cells_color(cells, "G") elif tag_name == "conflict": set_cells_color(cells, "R") return cells if add: p_rows.append(format_scores(p_score)) h_rows.append(format_scores(h_score)) p_score_list.append(p_score) h_score_list.append(h_score) pred_str = ["Entailment", "Neutral" , "Contradiction"][pred] out_entry = pred_str, p_tokens, h_tokens, p_score_list, h_score_list if len(selected_instances[pred]) < num_select : selected_instances[pred].append(out_entry) visualizer.write_headline(pred_str) visualizer.multirow_print_from_cells_list(p_rows) visualizer.multirow_print_from_cells_list(h_rows) visualizer.write_instance(pred_str, p_rows, h_rows) tex_visulizer.write_paragraph(str(pred)) tex_visulizer.multirow_print_from_cells_list(p_rows, width=13) tex_visulizer.multirow_print_from_cells_list(h_rows, width=13) if all([len(s) == num_select for s in selected_instances]): break tex_visulizer.close_table() return selected_instances
def doit(filename): name = filename.split(".")[0] bin_fn, mean_d, std_d = statistics_tlm() def get_score(p1, p2): key = bin_fn(p1) v = min(p2, p1) return (v - mean_d[key]) / std_d[key] st_list = [] ed_list = [] std_list = [] mean_list = [] for key in mean_d: st, ed = key st_list.append(st) ed_list.append(ed) std_list.append(std_d[key]) mean_list.append(mean_d[key]) mean_list = np.expand_dims(np.array(mean_list), 0) std_list = np.expand_dims(np.array(std_list), 0) st_list = np.expand_dims(np.array(st_list), 0) ed_list = np.expand_dims(np.array(ed_list), 0) def get_scores_lin(prob1_list, prob2_list): v2 = np.min(np.stack([prob1_list, prob2_list], axis=1), axis=1) v2 = np.expand_dims(v2, 1) all_scores = (v2 - mean_list) / std_list prob1_list = np.expand_dims(prob1_list, 1) f1 = np.less_equal(st_list, prob1_list) f2 = np.less(prob1_list, ed_list) f = np.logical_and(f1, f2) all_scores = all_scores * f scores = np.sum(all_scores, axis=1) return scores data = EstimatorPredictionViewerGosford(filename) amp = 0.5 html_writer = HtmlVisualizer("{}_{}.html".format(name, amp), dark_mode=False) for inst_i, entry in enumerate(data): if inst_i > 10: break tokens = entry.get_mask_resolved_input_mask_with_input() scores = entry.get_vector("priority_score") loss1 = entry.get_vector("lm_loss1") loss2 = entry.get_vector("lm_loss2") #scores1 = get_scores_lin(loss_to_prob(loss1), loss_to_prob(loss2)) #scores = [get_score(v1, v2) for v1,v2 in zip(loss_to_prob(loss1), loss_to_prob(loss2))] #assert np.all(np.less(np.abs(scores - scores1), 0.01)) prob_scores = probabilty(scores, amp) prob_strs = ["{:06.6f}".format(v*1000) for v in prob_scores] def normalize(prob): # 0-> Good # -1 -> Bad return min(prob * 10000, 100) norm_scores = lmap(normalize, prob_scores) cells = data.cells_from_tokens(tokens, norm_scores) cells2 = data.cells_from_anything(scores, norm_scores) cells3 = data.cells_from_anything(prob_strs, norm_scores) cells4 = data.cells_from_anything(loss_to_prob(loss1), norm_scores) cells5 = data.cells_from_anything(loss_to_prob(loss2), norm_scores) html_writer.multirow_print_from_cells_list([cells, cells2, cells3, cells4, cells5]) html_writer.write_headline("")