def per_doc_score(): filename = "tlm_view.pickle" html_writer = HtmlVisualizer("per_doc_score.html", dark_mode=False) data = EstimatorPredictionViewerGosford(filename) amp = 20 small_threshold = 40 for inst_i, entry in enumerate(data): if inst_i > 1000: break scores = entry.get_vector("priority_score") tokens = entry.get_mask_resolved_input_mask_with_input() cells = data.cells_from_tokens(tokens) if len(cells) < small_threshold: continue avg_score = average(scores) if -0.11 > avg_score > -0.30: continue print(average(scores)) html_writer.write_headline(avg_score) rows = [] row = [] for idx, cell in enumerate(cells): row.append(cell) if len(row) == 20: html_writer.write_table([row]) row = []
def dev(): train_data_feeder = load_cache("train_data_feeder") tokenizer = tokenizer_wo_tf.FullTokenizer( os.path.join(data_path, "bert_voca.txt")) html_writer = HtmlVisualizer("nli_w_dict.html", dark_mode=False) for _ in range(100): batch = train_data_feeder.get_random_batch(1) input_ids, input_mask, segment_ids, d_input_ids, d_input_mask, d_location_ids, y = batch tokens = tokenizer.convert_ids_to_tokens(input_ids[0]) for i in range(len(tokens)): if i is not 0 and i in d_location_ids: tokens[i] = "<b>{}</b>".format(tokens[i]) if tokens[i] == "[unused3]": tokens[i] = "[SEP]\n" s = tokenizer_wo_tf.pretty_tokens(tokens) html_writer.write_headline("Input") html_writer.write_paragraph(s) d_tokens = tokenizer.convert_ids_to_tokens(d_input_ids[0]) for i in range(len(d_tokens)): if tokens[i] == "[unused5]": tokens[i] = "<br>\n" s = tokenizer_wo_tf.pretty_tokens(d_tokens) html_writer.write_headline("Dict def") html_writer.write_paragraph(s) html_writer.close()
def visualize_prediction_data(data_id): tokenizer = get_tokenizer() num_samples_list = open( os.path.join(working_path, "entry_prediction_n", data_id), "r").readlines() p = os.path.join(working_path, "entry_loss", "entry{}.pickle".format(data_id)) loss_outputs_list = pickle.load(open(p, "rb")) print("Loaded input data") loss_outputs = [] for e in loss_outputs_list: loss_outputs.extend(e["masked_lm_example_loss"]) print("Total of {} loss outputs".format(len(loss_outputs))) instance_idx = 0 feature_itr = load_record_v2( os.path.join(working_path, "entry_prediction_tf.done", data_id)) n = len(num_samples_list) n = 100 html = HtmlVisualizer("entry_prediction.html") for i in range(n): n_sample = int(num_samples_list[i]) assert n_sample > 0 first_inst = feature_itr.__next__() feature = Feature2Text(first_inst, tokenizer) html.write_headline("Input:") html.write_paragraph(feature.get_input_as_text(True, True)) html.write_headline("Word:" + feature.get_selected_word_text()) if instance_idx + n_sample >= len(loss_outputs): break if n_sample == 1: continue rows = [] no_dict_loss = loss_outputs[instance_idx] row = [Cell(no_dict_loss, 0), Cell("")] rows.append(row) instance_idx += 1 for j in range(1, n_sample): feature = Feature2Text(feature_itr.__next__(), tokenizer) def_cell = Cell(feature.get_def_as_text()) loss = loss_outputs[instance_idx] hl_score = 100 if loss < no_dict_loss * 0.9 else 0 row = [Cell(loss, hl_score), def_cell] rows.append(row) instance_idx += 1 html.write_table(rows)
def show(all_info): html = HtmlVisualizer("cppnc.html") cnt = 0 for cpid, value in all_info.items(): score, rel_score, info = value[0] html.write_headline("Claim {}: {}".format(info['cid'], info['c_text'])) html.write_headline("Perspective: " + info['p_text']) for score, rel_score, info in value: html.write_headline("score: {}".format(score)) html.write_headline("rel_score: {}".format(rel_score)) html.write_paragraph(" ".join(info['passage'])) cnt += 1 if cnt > 10000: break
def main(): print("Loading scores...") cid_grouped: Dict[str, Dict[str, List[Dict]]] = load_cppnc_score_wrap() baseline_cid_grouped = load_baseline("train_baseline") gold = get_claim_perspective_id_dict() tokenizer = get_tokenizer() claim_d = load_train_claim_d() print("Start analyzing") html = HtmlVisualizer("cppnc_value_per_token_score.html") claim_cnt = 0 for cid, pid_entries_d in cid_grouped.items(): pid_entries_d: Dict[str, List[Dict]] = pid_entries_d pid_entries: List[Tuple[str, List[Dict]]] = list(pid_entries_d.items()) baseline_pid_entries = baseline_cid_grouped[int(cid)] baseline_score_d = fetch_score_per_pid(baseline_pid_entries) gold_pids = gold[int(cid)] ret = collect_score_per_doc(baseline_score_d, get_score_from_entry, gold_pids, pid_entries) passage_tokens_d = collect_passage_tokens(pid_entries) doc_info_d: Dict[int, Tuple[str, int]] = ret[0] doc_value_arr: List[List[float]] = ret[1] kdp_result_grouped = defaultdict(list) for doc_idx, doc_values in enumerate(doc_value_arr): doc_id, passage_idx = doc_info_d[doc_idx] avg_score = average(doc_values) kdp_result = doc_id, passage_idx, avg_score kdp_result_grouped[doc_id].append(kdp_result) s = "{} : {}".format(cid, claim_d[int(cid)]) html.write_headline(s) claim_cnt += 1 if claim_cnt > 10: break scores: List[float] = list([r[2] for r in doc_value_arr]) foreach(html.write_paragraph, lmap(str, scores)) for doc_id, kdp_result_list in kdp_result_grouped.items(): html.write_headline(doc_id) tokens, per_token_score = combine_collect_score(tokenizer, doc_id, passage_tokens_d, kdp_result_list) str_tokens = tokenizer.convert_ids_to_tokens(tokens) row = cells_from_tokens(str_tokens) for idx in range(len(str_tokens)): score = per_token_score[idx][0] norm_score = min(abs(score) * 10000, 100) color = "B" if score > 0 else "R" row[idx].highlight_score = norm_score row[idx].target_color = color rows = [row] nth = 0 any_score_found = True while any_score_found: any_score_found = False score_list = [] for idx in range(len(str_tokens)): if nth < len(per_token_score[idx]): score = per_token_score[idx][nth] any_score_found = True else: score = "-" score_list.append(score) def get_cell(score): if score == "-": return Cell("-") else: # 0.01 -> 100 norm_score = min(abs(score) * 10000, 100) color = "B" if score > 0 else "R" return Cell("", highlight_score=norm_score, target_color=color) nth += 1 if any_score_found: row = lmap(get_cell, score_list) rows.append(row) html.multirow_print_from_cells_list(rows)
def analyze_gradient(data, tokenizer): gradients = data['gradients'] d_input_ids = data['d_input_ids'] mask_input_ids = data['masked_input_ids'] masked_lm_positions = data["masked_lm_positions"] n_inst, seq_len = mask_input_ids.shape n_inst2, def_len = d_input_ids.shape assert n_inst == n_inst2 def_len = 256 hidden_dim = 768 reshaped_grad = reshape_gradienet(gradients, n_inst, def_len, hidden_dim) print(reshaped_grad.shape) n_pred = reshaped_grad.shape[1] grad_per_token = np.sum(np.abs(reshaped_grad), axis=3) html_writer = HtmlVisualizer("dict_grad.html", dark_mode=False) for inst_idx in range(n_inst): tokens = tokenizer.convert_ids_to_tokens(mask_input_ids[inst_idx]) #ans_tokens = tokenizer.convert_ids_to_tokens(input_ids[inst_idx]) for i in range(len(tokens)): if tokens[i] == "[MASK]": tokens[i] = "[MASK_{}]".format(i) if tokens[i] == "[SEP]": tokens[i] = "[SEP]<br>" def_tokens = tokenizer.convert_ids_to_tokens(d_input_ids[inst_idx]) s = tokenizer_wo_tf.pretty_tokens(tokens) lines = [] grad_total_max = 0 for pred_idx in range(n_pred): row = [] max_val = max(grad_per_token[inst_idx, pred_idx]) total = sum(grad_per_token[inst_idx, pred_idx]) mask_pos = masked_lm_positions[inst_idx, pred_idx] if total > grad_total_max: grad_total_max = total row.append(Cell(mask_pos)) row.append(Cell(int(total))) for def_idx in range(def_len): term = def_tokens[def_idx] cont_right = def_idx + 1 < def_len and def_tokens[ def_idx][:2] == "##" cont_left = term[:2] == "##" space_left = " " if not cont_left else "" space_right = " " if not cont_right else "" if term == "[PAD]": break if term == "[unused5]": term = "[\\n]" score = grad_per_token[inst_idx, pred_idx, def_idx] / (hidden_dim * 2) bg_color = get_color(score) row.append(Cell(term, score, not cont_left, not cont_right)) print("{}({})".format( term, grad_per_token[inst_idx, pred_idx, def_idx]), end=" ") lines.append((mask_pos, row)) print("") lines.sort(key=lambda x: x[0]) s = s.replace("[unused4]", "<b>DictTerm</b>") html_writer.write_paragraph(s) if grad_total_max > 5000000: html_writer.write_headline("HIGH Gradient") rows = right(lines) html_writer.write_table(rows) print("----------") html_writer.close()
def doc_lm_scoring(): gold = get_claim_perspective_id_dict() d_ids = list(load_train_claim_ids()) claims: List[Dict] = get_claims_from_ids(d_ids) claims = claims top_n = 10 q_res_path = FilePath( "/mnt/nfs/work3/youngwookim/data/perspective/train_claim/q_res_100") ranked_list: Dict[ str, List[SimpleRankedListEntry]] = load_galago_ranked_list(q_res_path) preload_docs(ranked_list, claims, top_n) claim_lms = build_gold_lms(claims) claim_lms_d = {lm.cid: lm for lm in claim_lms} bg_lm = average_counters(lmap(lambda x: x.LM, claim_lms)) log_bg_lm = get_lm_log(bg_lm) stopwords = load_stopwords_for_query() alpha = 0.5 html_visualizer = HtmlVisualizer("doc_lm_doc_level.html") tokenizer = PCTokenizer() random_passages = [] num_pos_sum = 0 num_pos_exists = 0 for c in claims: q_res: List[SimpleRankedListEntry] = ranked_list[str(c['cId'])] html_visualizer.write_headline("{} : {}".format(c['cId'], c['text'])) # for cluster in clusters: # html_visualizer.write_paragraph("---") # p_text_list: List[str] = lmap(perspective_getter, cluster) # for text in p_text_list: # html_visualizer.write_paragraph(text) # html_visualizer.write_paragraph("---") claim_lm = claim_lms_d[c['cId']] topic_lm_prob = smooth(claim_lm.LM, bg_lm, alpha) log_topic_lm = get_lm_log(smooth(claim_lm.LM, bg_lm, alpha)) log_odd: Counter = subtract(log_topic_lm, log_bg_lm) claim_text = c['text'] claim_tokens = tokenizer.tokenize_stem(claim_text) scores = [] for t in claim_tokens: if t in log_odd: scores.append(log_odd[t]) threshold = average(scores) s = "\t".join(left(log_odd.most_common(30))) html_visualizer.write_paragraph("Log odd top: " + s) not_found = set() def get_log_odd(x): x = tokenizer.stemmer.stem(x) if x not in log_odd: not_found.add(x) return log_odd[x] def get_probs(x): x = tokenizer.stemmer.stem(x) if x not in topic_lm_prob: not_found.add(x) return topic_lm_prob[x] def get_passage_score(p): return sum([log_odd[tokenizer.stemmer.stem(t)] for t in p]) / len(p) if len(p) > 0 else 0 passages = iterate_passages(q_res, top_n, get_passage_score) passages.sort(key=lambda x: x[1], reverse=True) html_visualizer.write_paragraph("Threshold {}".format(threshold)) top5_scores = right(passages[:5]) bot5_scores = right(passages[-5:]) if len(random_passages) > 5: random_sel_pssages = random.choices(random_passages, k=5) else: random_sel_pssages = [] random5_scores = lmap(get_passage_score, random_sel_pssages) def score_line(scores): return " ".join(lmap(two_digit_float, scores)) html_visualizer.write_paragraph("top 5: " + score_line(top5_scores)) html_visualizer.write_paragraph("bot 5: " + score_line(bot5_scores)) html_visualizer.write_paragraph("random 5: " + score_line(random5_scores)) num_pos = len(lfilter(lambda x: x[1] > 0, passages)) num_pos_sum += num_pos if num_pos > 0: num_pos_exists += 1 def print_doc(doc, html_visualizer, score): cells = lmap(lambda x: get_cell_from_token(x, get_log_odd(x)), doc) html_visualizer.write_headline("score={}".format(score)) html_visualizer.multirow_print(cells, width=20) random_passages.extend(left(passages)) if threshold < 0: continue for doc, score in passages: if score < 0: break print_doc(doc, html_visualizer, score) html_visualizer.write_headline("Bottom 5") for doc, score in passages[-5:]: print_doc(doc, html_visualizer, score) print("{} claims. {} docs on {} claims".format(len(claims), num_pos_sum, num_pos_exists))
def join_docs_and_lm(): gold = get_claim_perspective_id_dict() d_ids = list(load_train_claim_ids()) claims: List[Dict] = get_claims_from_ids(d_ids) claims = claims[:10] top_n = 10 q_res_path = FilePath( "/mnt/nfs/work3/youngwookim/data/perspective/train_claim/q_res_100") ranked_list: Dict[ str, List[SimpleRankedListEntry]] = load_galago_ranked_list(q_res_path) preload_docs(ranked_list, claims, top_n) claim_lms = build_gold_lms(claims) claim_lms_d = {lm.cid: lm for lm in claim_lms} bg_lm = average_counters(lmap(lambda x: x.LM, claim_lms)) log_bg_lm = get_lm_log(bg_lm) stopwords.update([".", ",", "!", "?"]) alpha = 0.1 html_visualizer = HtmlVisualizer("doc_lm_joined.html") def get_cell_from_token2(token, probs): if token.lower() in stopwords: probs = 0 probs = probs * 1e5 s = min(100, probs) c = Cell(token, s) return c tokenizer = PCTokenizer() for c in claims: q_res: List[SimpleRankedListEntry] = ranked_list[str(c['cId'])] html_visualizer.write_headline("{} : {}".format(c['cId'], c['text'])) clusters: List[List[int]] = gold[c['cId']] for cluster in clusters: html_visualizer.write_paragraph("---") p_text_list: List[str] = lmap(perspective_getter, cluster) for text in p_text_list: html_visualizer.write_paragraph(text) html_visualizer.write_paragraph("---") claim_lm = claim_lms_d[c['cId']] topic_lm_prob = smooth(claim_lm.LM, bg_lm, alpha) log_topic_lm = get_lm_log(smooth(claim_lm.LM, bg_lm, alpha)) log_odd: Counter = subtract(log_topic_lm, log_bg_lm) s = "\t".join(left(log_odd.most_common(30))) html_visualizer.write_paragraph("Log odd top: " + s) not_found = set() def get_log_odd(x): x = tokenizer.stemmer.stem(x) if x not in log_odd: not_found.add(x) return log_odd[x] def get_probs(x): x = tokenizer.stemmer.stem(x) if x not in topic_lm_prob: not_found.add(x) return topic_lm_prob[x] for i in range(top_n): try: doc = load_doc(q_res[i].doc_id) cells = lmap(lambda x: get_cell_from_token(x, get_log_odd(x)), doc) html_visualizer.write_headline("Doc rank {}".format(i)) html_visualizer.multirow_print(cells, width=20) except KeyError: pass html_visualizer.write_paragraph("Not found: {}".format(not_found))
def show_analyzed_html(analyzed_failture_cases: List[AnalyzedCase]): tokenizer = get_tokenizer() html = HtmlVisualizer("ca_contradiction_tokens.html") def get_token_scored_cells(sent1, sent2, token_score): tokens1 = tokenizer.tokenize(sent1) tokens2 = tokenizer.tokenize(sent2) print(token_score) score_for_1 = token_score[1:1 + len(tokens1)] score_for_2 = token_score[2 + len(tokens1):2 + len(tokens1) + len(tokens2)] assert len(tokens1) == len(score_for_1) assert len(tokens2) == len(score_for_2) def get_cells(tokens, scores): cap = max(max(scores), 1) factor = 100 / cap def normalize_score(s): return min(s * factor, 100) return list( [Cell(t, normalize_score(s)) for t, s in zip(tokens, scores)]) cells1 = get_cells(tokens1, score_for_1) cells2 = get_cells(tokens2, score_for_2) return cells1, cells2 def print_scored_sentences(scores): for i, _ in enumerate(scores): if i % 2 == 0: sent1, sent2, score1, token_score1 = scores[i] _, _, score2, token_score2 = scores[i + 1] if is_cont(score1): cells1, cells2 = get_token_scored_cells( sent1, sent2, token_score1) html.write_paragraph( "Forward, P(Contradiction) = {}".format(score1[2])) html.write_table([cells1]) html.write_table([cells2]) if is_cont(score2): cells1, cells2 = get_token_scored_cells( sent2, sent1, token_score2) html.write_paragraph( "Backward, P(Contradiction) = {}".format(score2[2])) html.write_table([cells1]) html.write_table([cells2]) def print_analyzed_case(analyzed_case: AnalyzedCase): def print_part(score_list): cnt = count_cont(score_list) s = "{} of {}".format(cnt, len(score_list)) html.write_paragraph(s) print_scored_sentences(score_list) html.write_paragraph("Gold") print_part(analyzed_case.score_g) html.write_paragraph("Pred") print_part(analyzed_case.score_p) def is_cont(scores): return np.argmax(scores) == 2 def is_cont_strict(scores): return scores[2] > 0.9 def count_cont(result_list): num_cont = sum( [1 for _, _, scores, _ in result_list if is_cont(scores)]) for idx, dp in enumerate(analyzed_failture_cases): html.write_headline("Data point {}".format(idx)) html.write_paragraph("------------") print_analyzed_case(dp)
def show_all(run_name, data_id): num_tags = 3 num_select = 1 pickle_name = "save_view_{}_{}".format(run_name, data_id) tokenizer = get_tokenizer() data_loader = get_modified_data_loader2(HPSENLI3(), NLIExTrainConfig()) explain_entries = load_from_pickle(pickle_name) explain_entries = explain_entries visualizer = HtmlVisualizer(pickle_name + ".html") tex_visulizer = TexTableNLIVisualizer(pickle_name + ".tex") tex_visulizer.begin_table() selected_instances = [[], [], []] for idx, entry in enumerate(explain_entries): x0, logits, scores = entry pred = np.argmax(logits) input_ids = x0 p, h = data_loader.split_p_h_with_input_ids(input_ids, input_ids) p_tokens = tokenizer.convert_ids_to_tokens(p) h_tokens = tokenizer.convert_ids_to_tokens(h) p_rows = [] h_rows = [] p_rows.append(cells_from_tokens(p_tokens)) h_rows.append(cells_from_tokens(h_tokens)) p_score_list = [] h_score_list = [] for j in range(num_tags): tag_name = data_generator.NLI.nli_info.tags[j] p_score, h_score = data_loader.split_p_h_with_input_ids(scores[j], input_ids) normalize_fn = normalize add = True if pred == "0": add = tag_name == "match" if pred == "1": add = tag_name == "mismatch" if pred == "2": add = tag_name == "conflict" def format_scores(raw_scores): def format_float(s): return "{0:.2f}".format(s) norm_scores = normalize_fn(raw_scores) cells = [Cell(format_float(s1), s2, False, False) for s1, s2 in zip(raw_scores, norm_scores)] if tag_name == "mismatch": set_cells_color(cells, "G") elif tag_name == "conflict": set_cells_color(cells, "R") return cells if add: p_rows.append(format_scores(p_score)) h_rows.append(format_scores(h_score)) p_score_list.append(p_score) h_score_list.append(h_score) pred_str = ["Entailment", "Neutral" , "Contradiction"][pred] out_entry = pred_str, p_tokens, h_tokens, p_score_list, h_score_list if len(selected_instances[pred]) < num_select : selected_instances[pred].append(out_entry) visualizer.write_headline(pred_str) visualizer.multirow_print_from_cells_list(p_rows) visualizer.multirow_print_from_cells_list(h_rows) visualizer.write_instance(pred_str, p_rows, h_rows) tex_visulizer.write_paragraph(str(pred)) tex_visulizer.multirow_print_from_cells_list(p_rows, width=13) tex_visulizer.multirow_print_from_cells_list(h_rows, width=13) if all([len(s) == num_select for s in selected_instances]): break tex_visulizer.close_table() return selected_instances
def load_bert_like(): disable_eager_execution() model = BertLike() sess = init_session() #sess.run(tf.compat.v1.global_variables_initializer()) load_v2_to_v2(sess, get_bert_full_path(), False) attention_prob_list, = sess.run([model.attention_probs_list]) html = HtmlVisualizer("position.html") for layer_no, attention_prob in enumerate(attention_prob_list): html.write_headline("Layer {}".format(layer_no)) acc_dict = {} zero_scores = [list() for _ in range(12)] for loc in range(2, 40, 2): print("Source : ", loc) for target_loc in range(20): offset = target_loc - loc print(offset, end=" ") for head_idx in range(num_head): key = offset, head_idx if key not in acc_dict: acc_dict[key] = [] e = attention_prob[0, head_idx, loc, target_loc] if target_loc != 0: acc_dict[key].append(e) else: zero_scores[head_idx].append(e) print("{0:.2f}".format(e * 100), end=" ") print() rows = [[Cell("Loc")] + [Cell("Head{}".format(i)) for i in range(12)]] for offset in range(-7, +7): print(offset, end=" ") scores = [] for head_idx in range(12): key = offset, head_idx try: elems = acc_dict[key] if len(elems) < 3: raise KeyError avg = average(elems) scores.append(avg) print("{0:.2f}".format(avg * 100), end=" ") except KeyError: print("SKIP") print() rows.append([Cell(offset)] + [Cell(float(v * 100), v * 1000) for v in scores]) html.write_table(rows) html.write_paragraph("Attention to first token") zero_scores = [average(l) for l in zero_scores] rows = [[Cell(" ")] + [Cell("Head{}".format(i)) for i in range(12)], [Cell(" ")] + [Cell(float(v * 100), v * 1000) for v in zero_scores]] html.write_table(rows)
def pred_loss_view(): tokenizer = get_tokenizer() filename = "tlm_loss_pred.pickle" filename = "tlm_loss_pred_on_dev.pickle" p = os.path.join(output_path, filename) data = pickle.load(open(p, "rb")) batch_size, seq_length = data[0]['input_ids'].shape keys = list(data[0].keys()) vectors = {} for e in data: for key in keys: if key not in vectors: vectors[key] = [] vectors[key].append(e[key]) for key in keys: vectors[key] = np.concatenate(vectors[key], axis=0) html_writer = HtmlVisualizer("pred_make_sense_dev.html", dark_mode=False) n_instance = len(vectors['input_ids']) n_instance = min(n_instance, 100) for inst_idx in range(n_instance): tokens = tokenizer.convert_ids_to_tokens( vectors['input_ids'][inst_idx]) locations = list(vectors['masked_lm_positions'][inst_idx]) def is_dependent(token): return len(token) == 1 and not token[0].isalnum() cells = [] for i in range(len(tokens)): f_same_pred = False score = 0 if i in locations and i != 0: i_idx = locations.index(i) tokens[i] = "[{}:{}]".format(i_idx, tokens[i]) pred_diff = vectors['pred_diff'][inst_idx][i_idx] gold_diff = vectors['gold_diff'][inst_idx][i_idx] pred_label = pred_diff > 0.3 gold_label = gold_diff > 0.3 if pred_label: score = 100 if gold_label: f_same_pred = True else: if gold_label: score = 30 f_same_pred = False if tokens[i] == "[SEP]": tokens[i] = "[SEP]<br>" if tokens[i] != "[PAD]": term = tokens[i] cont_left = term[:2] == "##" cont_right = i + 1 < len(tokens) and tokens[i + 1][:2] == "##" if i + 1 < len(tokens): dependent_right = is_dependent(tokens[i + 1]) else: dependent_right = False dependent_left = is_dependent(tokens[i]) if cont_left: term = term[2:] space_left = " " if not (cont_left or dependent_left) else "" space_right = " " if not (cont_right or dependent_right) else "" if f_same_pred: cells.append(Cell(term, score, space_left, space_right)) else: cells.append( Cell(term, score, space_left, space_right, target_color="R")) row = [] for cell in cells: row.append(cell) if len(row) == 20: html_writer.write_table([row]) row = [] row_head = [ Cell("Index"), Cell("P]Prob1"), Cell("P]Prob2"), Cell("G]Prob1"), Cell("G]Prob2"), Cell("P]Diff"), Cell("G]Diff"), ] def f_cell(obj): return Cell("{:04.2f}".format(obj)) rows = [row_head] pred_diff_list = [] gold_diff_list = [] for idx, pos in enumerate(locations): if pos == 0: break pred_diff = vectors['pred_diff'][inst_idx][idx] gold_diff = vectors['gold_diff'][inst_idx][idx] pred_diff_list.append(pred_diff) gold_diff_list.append(gold_diff) row = [ Cell(idx), f_cell(vectors['prob1'][inst_idx][idx]), f_cell(vectors['prob2'][inst_idx][idx]), f_cell(math.exp(-vectors['loss_base'][inst_idx][idx])), f_cell(math.exp(-vectors['loss_target'][inst_idx][idx])), f_cell(pred_diff), f_cell(gold_diff), ] rows.append(row) html_writer.write_table(rows) pred_diff = np.average(pred_diff_list) gold_diff = np.average(gold_diff_list) html_writer.write_paragraph( "Average Pred diff ={:04.2f} Observed diff={:04.2f} ".format( pred_diff, gold_diff)) if pred_diff > 0.3: html_writer.write_headline("High Drop") elif pred_diff < 0.1: html_writer.write_headline("Low Drop")
def do(): pred_file_name = "RLPP_0.pickle" pred_file_name = "ukp_rel.pickle" record_file_name = "C:\\work\\Code\\Chair\\output\\unmasked_pair_x3_0" record_file_name = "C:\\work\\Code\\Chair\\output\\tf_enc" todo = [ ("RLPP_0.pickle", "C:\\work\\Code\\Chair\\output\\unmasked_pair_x3_0", "RLPP_wiki.html"), ("ukp_rel.pickle", "C:\\work\\Code\\Chair\\output\\tf_enc", "RLPP_ukp.html") ] x = [] y = [] for pred_file_name, record_file_name, out_name in todo: viewer = EstimatorPredictionViewerGosford(pred_file_name) html = HtmlVisualizer(out_name) itr1 = load_record_v2(record_file_name) itr2 = viewer.__iter__() cnt = 0 for features, entry in zip(itr1, itr2): cnt += 1 if cnt > 200: break input_ids1 = entry.get_tokens("input_ids") prob1 = entry.get_vector("prob1") prob2 = entry.get_vector("prob2") cells = viewer.cells_from_tokens(input_ids1) p1_l = [] p2_l = [] useful_l = [] row1 = [] row2 = [] row3 = [] row4 = [] for j, cell in enumerate(cells): p1 = float(prob1[j]) p2 = float(prob2[j]) x.append([p1]) y.append(p2) u = useful(p1, p2) score = (1 - u) * 100 cell.highlight_score = score row1.append(cell) row2.append(Cell(p1, score)) row3.append(Cell(p2, score)) row4.append(Cell(u, score)) p1_l.append(p1) p2_l.append(p2) useful_l.append(u) if len(row1) > 20: rows = [row1, row2, row3, row4] row1 = [] row2 = [] row3 = [] row4 = [] html.write_table(rows) html.write_paragraph("p1: {}".format(average(p1_l))) html.write_paragraph("p2: {}".format(average(p2_l))) html.write_paragraph("useful: {}".format(average(useful_l))) if average(useful_l) < 0.4: html.write_headline("Low Score") l = list(zip(x, y)) random.shuffle(l) l = l[:1000] x, y = zip(*l) lin = LinearRegression() lin.fit(x, y) poly = PolynomialFeatures(degree=4) X_poly = poly.fit_transform(x) poly.fit(X_poly, y) lin2 = LinearRegression() lin2.fit(X_poly, y) plt.scatter(x, y, color='blue') plt.plot(x, lin2.predict(poly.fit_transform(x)), color='red') plt.title('Polynomial Regression') plt.show()
def doit(filename): name = filename.split(".")[0] bin_fn, mean_d, std_d = statistics_tlm() def get_score(p1, p2): key = bin_fn(p1) v = min(p2, p1) return (v - mean_d[key]) / std_d[key] st_list = [] ed_list = [] std_list = [] mean_list = [] for key in mean_d: st, ed = key st_list.append(st) ed_list.append(ed) std_list.append(std_d[key]) mean_list.append(mean_d[key]) mean_list = np.expand_dims(np.array(mean_list), 0) std_list = np.expand_dims(np.array(std_list), 0) st_list = np.expand_dims(np.array(st_list), 0) ed_list = np.expand_dims(np.array(ed_list), 0) def get_scores_lin(prob1_list, prob2_list): v2 = np.min(np.stack([prob1_list, prob2_list], axis=1), axis=1) v2 = np.expand_dims(v2, 1) all_scores = (v2 - mean_list) / std_list prob1_list = np.expand_dims(prob1_list, 1) f1 = np.less_equal(st_list, prob1_list) f2 = np.less(prob1_list, ed_list) f = np.logical_and(f1, f2) all_scores = all_scores * f scores = np.sum(all_scores, axis=1) return scores data = EstimatorPredictionViewerGosford(filename) amp = 0.5 html_writer = HtmlVisualizer("{}_{}.html".format(name, amp), dark_mode=False) for inst_i, entry in enumerate(data): if inst_i > 10: break tokens = entry.get_mask_resolved_input_mask_with_input() scores = entry.get_vector("priority_score") loss1 = entry.get_vector("lm_loss1") loss2 = entry.get_vector("lm_loss2") #scores1 = get_scores_lin(loss_to_prob(loss1), loss_to_prob(loss2)) #scores = [get_score(v1, v2) for v1,v2 in zip(loss_to_prob(loss1), loss_to_prob(loss2))] #assert np.all(np.less(np.abs(scores - scores1), 0.01)) prob_scores = probabilty(scores, amp) prob_strs = ["{:06.6f}".format(v*1000) for v in prob_scores] def normalize(prob): # 0-> Good # -1 -> Bad return min(prob * 10000, 100) norm_scores = lmap(normalize, prob_scores) cells = data.cells_from_tokens(tokens, norm_scores) cells2 = data.cells_from_anything(scores, norm_scores) cells3 = data.cells_from_anything(prob_strs, norm_scores) cells4 = data.cells_from_anything(loss_to_prob(loss1), norm_scores) cells5 = data.cells_from_anything(loss_to_prob(loss2), norm_scores) html_writer.multirow_print_from_cells_list([cells, cells2, cells3, cells4, cells5]) html_writer.write_headline("")