def draw2(in_file, out_file):
filename = os.path.join(output_path, in_file)
data = EstimatorPredictionViewerGosford(filename)
html_writer = HtmlVisualizer(out_file, dark_mode=False)
tokenizer = get_tokenizer()
for inst_i, entry in enumerate(data):
if inst_i > 100:
break
tokens = entry.get_tokens("input_ids")
# tokens = entry.get_tokens("input_ids")
prob1 = entry.get_vector("prob1")
prob2 = entry.get_vector("prob2")
real_loss1 = entry.get_vector("per_example_loss1")
real_loss2 = entry.get_vector("per_example_loss2")
masked_lm_positions = entry.get_vector("masked_lm_positions")
for i, loc in enumerate(masked_lm_positions):
tokens[loc] = "[{}:{}]".format(i, tokens[loc])
html_writer.multirow_print(data.cells_from_tokens(tokens))
row2 = [Cell("prob1:")] + data.cells_from_anything(prob1)
row3 = [Cell("prob2:")] + data.cells_from_anything(prob2)
row4 = [Cell("real_loss1:")] + data.cells_from_anything(real_loss1)
row5 = [Cell("real_loss2:")] + data.cells_from_anything(real_loss2)
html_writer.multirow_print_from_cells_list([row2, row3, row4, row5])
def view_grad_overlap_hidden():
filename = "ukp_feature_overlap.pickle"
obj = pickle.load(open(os.path.join(output_path, filename), "rb"))
out_name = filename.split(".")[0] + ".html"
html_writer = HtmlVisualizer(out_name, dark_mode=False)
data = EstimatorPredictionViewerGosford(filename)
for inst_i, entry in enumerate(data):
tokens = entry.get_mask_resolved_input_mask_with_input()
h_overlap = entry.get_vector('h_overlap')
std = np.std(h_overlap, axis=2)
mean = np.mean(h_overlap, axis=2)
h_overlap = np.sum(h_overlap, axis=2)
highlight = lmap(is_mask, tokens)
cells = data.cells_from_tokens(tokens, highlight)
rows = [cells]
for layer_i in range(12):
e = h_overlap[layer_i, :]
e = [v * 1e6 for v in e]
cells = data.cells_from_scores(e)
rows.append(cells)
e = [v * 1e8 for v in std[layer_i, :]]
cells2 = data.cells_from_scores(e)
rows.append(cells2)
print(entry.get_vector("masked_lm_example_loss"))
html_writer.multirow_print_from_cells_list(rows, 40)
def main():
file_path = sys.argv[1]
name = os.path.basename(file_path)
viewer = EstimatorPredictionViewer(file_path)
html = HtmlVisualizer("toke_score_gold.html")
stopwords = load_stopwords_for_query()
skip = 10
for entry_idx, entry in enumerate(viewer):
if entry_idx % skip != 0:
continue
tokens = entry.get_tokens("input_ids")
input_ids = entry.get_vector("input_ids")
label_ids = entry.get_vector("label_ids")
label_ids = np.reshape(label_ids, [-1, 2])
log_label_ids = np.log(label_ids + 1e-10)
seg1, seg2 = split_p_h_with_input_ids(tokens, input_ids)
pad_idx = tokens.index("[PAD]")
assert pad_idx > 0
logits = entry.get_vector("logits")
cells = []
cells2 = []
for idx in range(pad_idx):
probs = label_ids[idx]
token = tokens[idx]
score = probs[0]
color = "B" if score > 0 else "R"
highlight_score = min(abs(score) * 10000, 100)
if token in stopwords:
highlight_score = 0
if token in seg1:
highlight_score = 50
color = "G"
c = Cell(token,
highlight_score=highlight_score,
target_color=color)
cells.append(c)
html.multirow_print_from_cells_list([cells, cells2])
if entry_idx > 10000:
break
def show(filename):
data = EstimatorPredictionViewerGosford(filename)
html_writer = HtmlVisualizer("token_scoring.html", dark_mode=False)
correctness = []
for entry in data:
tokens = entry.get_tokens("input_ids")
logits = entry.get_vector("logits")
masks = entry.get_vector("label_masks")
ids = entry.get_vector("labels")
token_row = []
pred_row = []
gold_row = []
rows = [token_row, pred_row, gold_row]
for idx, token in enumerate(tokens):
token_cell = Cell(token)
if token == "[PAD]":
break
model_score = logits[idx][0]
if masks[idx]:
correct = (model_score > 0 and ids[idx] > 0) or (model_score < 0 and ids[idx] < 0)
color = "B" if correct else "R"
if correct and (model_score > 0 and ids[idx] > 0) :
color = "G"
pred_cell = Cell("{0:.2f}".format(model_score), 100, target_color=color)
gold_cell = Cell("{0:.2f}".format(ids[idx]), 100, target_color=color)
else:
token_cell = Cell(token)
pred_cell = Cell("")
gold_cell = Cell("")
token_row.append(token_cell)
pred_row.append(pred_cell)
gold_row.append(gold_cell)
html_writer.multirow_print_from_cells_list(rows, 20)
def main():
print("Loading scores...")
cid_grouped: Dict[str, Dict[str, List[Dict]]] = load_cppnc_score_wrap()
baseline_cid_grouped = load_baseline("train_baseline")
gold = get_claim_perspective_id_dict()
tokenizer = get_tokenizer()
claim_d = load_train_claim_d()
print("Start analyzing")
html = HtmlVisualizer("cppnc_value_per_token_score.html")
claim_cnt = 0
for cid, pid_entries_d in cid_grouped.items():
pid_entries_d: Dict[str, List[Dict]] = pid_entries_d
pid_entries: List[Tuple[str, List[Dict]]] = list(pid_entries_d.items())
baseline_pid_entries = baseline_cid_grouped[int(cid)]
baseline_score_d = fetch_score_per_pid(baseline_pid_entries)
gold_pids = gold[int(cid)]
ret = collect_score_per_doc(baseline_score_d, get_score_from_entry, gold_pids,
pid_entries)
passage_tokens_d = collect_passage_tokens(pid_entries)
doc_info_d: Dict[int, Tuple[str, int]] = ret[0]
doc_value_arr: List[List[float]] = ret[1]
kdp_result_grouped = defaultdict(list)
for doc_idx, doc_values in enumerate(doc_value_arr):
doc_id, passage_idx = doc_info_d[doc_idx]
avg_score = average(doc_values)
kdp_result = doc_id, passage_idx, avg_score
kdp_result_grouped[doc_id].append(kdp_result)
s = "{} : {}".format(cid, claim_d[int(cid)])
html.write_headline(s)
claim_cnt += 1
if claim_cnt > 10:
break
scores: List[float] = list([r[2] for r in doc_value_arr])
foreach(html.write_paragraph, lmap(str, scores))
for doc_id, kdp_result_list in kdp_result_grouped.items():
html.write_headline(doc_id)
tokens, per_token_score = combine_collect_score(tokenizer, doc_id, passage_tokens_d, kdp_result_list)
str_tokens = tokenizer.convert_ids_to_tokens(tokens)
row = cells_from_tokens(str_tokens)
for idx in range(len(str_tokens)):
score = per_token_score[idx][0]
norm_score = min(abs(score) * 10000, 100)
color = "B" if score > 0 else "R"
row[idx].highlight_score = norm_score
row[idx].target_color = color
rows = [row]
nth = 0
any_score_found = True
while any_score_found:
any_score_found = False
score_list = []
for idx in range(len(str_tokens)):
if nth < len(per_token_score[idx]):
score = per_token_score[idx][nth]
any_score_found = True
else:
score = "-"
score_list.append(score)
def get_cell(score):
if score == "-":
return Cell("-")
else:
# 0.01 -> 100
norm_score = min(abs(score) * 10000, 100)
color = "B" if score > 0 else "R"
return Cell("", highlight_score=norm_score, target_color=color)
nth += 1
if any_score_found:
row = lmap(get_cell, score_list)
rows.append(row)
html.multirow_print_from_cells_list(rows)
def show_all(run_name, data_id):
num_tags = 3
num_select = 1
pickle_name = "save_view_{}_{}".format(run_name, data_id)
tokenizer = get_tokenizer()
data_loader = get_modified_data_loader2(HPSENLI3(), NLIExTrainConfig())
explain_entries = load_from_pickle(pickle_name)
explain_entries = explain_entries
visualizer = HtmlVisualizer(pickle_name + ".html")
tex_visulizer = TexTableNLIVisualizer(pickle_name + ".tex")
tex_visulizer.begin_table()
selected_instances = [[], [], []]
for idx, entry in enumerate(explain_entries):
x0, logits, scores = entry
pred = np.argmax(logits)
input_ids = x0
p, h = data_loader.split_p_h_with_input_ids(input_ids, input_ids)
p_tokens = tokenizer.convert_ids_to_tokens(p)
h_tokens = tokenizer.convert_ids_to_tokens(h)
p_rows = []
h_rows = []
p_rows.append(cells_from_tokens(p_tokens))
h_rows.append(cells_from_tokens(h_tokens))
p_score_list = []
h_score_list = []
for j in range(num_tags):
tag_name = data_generator.NLI.nli_info.tags[j]
p_score, h_score = data_loader.split_p_h_with_input_ids(scores[j], input_ids)
normalize_fn = normalize
add = True
if pred == "0":
add = tag_name == "match"
if pred == "1":
add = tag_name == "mismatch"
if pred == "2":
add = tag_name == "conflict"
def format_scores(raw_scores):
def format_float(s):
return "{0:.2f}".format(s)
norm_scores = normalize_fn(raw_scores)
cells = [Cell(format_float(s1), s2, False, False) for s1, s2 in zip(raw_scores, norm_scores)]
if tag_name == "mismatch":
set_cells_color(cells, "G")
elif tag_name == "conflict":
set_cells_color(cells, "R")
return cells
if add:
p_rows.append(format_scores(p_score))
h_rows.append(format_scores(h_score))
p_score_list.append(p_score)
h_score_list.append(h_score)
pred_str = ["Entailment", "Neutral" , "Contradiction"][pred]
out_entry = pred_str, p_tokens, h_tokens, p_score_list, h_score_list
if len(selected_instances[pred]) < num_select :
selected_instances[pred].append(out_entry)
visualizer.write_headline(pred_str)
visualizer.multirow_print_from_cells_list(p_rows)
visualizer.multirow_print_from_cells_list(h_rows)
visualizer.write_instance(pred_str, p_rows, h_rows)
tex_visulizer.write_paragraph(str(pred))
tex_visulizer.multirow_print_from_cells_list(p_rows, width=13)
tex_visulizer.multirow_print_from_cells_list(h_rows, width=13)
if all([len(s) == num_select for s in selected_instances]):
break
tex_visulizer.close_table()
return selected_instances
def doit(filename):
name = filename.split(".")[0]
bin_fn, mean_d, std_d = statistics_tlm()
def get_score(p1, p2):
key = bin_fn(p1)
v = min(p2, p1)
return (v - mean_d[key]) / std_d[key]
st_list = []
ed_list = []
std_list = []
mean_list = []
for key in mean_d:
st, ed = key
st_list.append(st)
ed_list.append(ed)
std_list.append(std_d[key])
mean_list.append(mean_d[key])
mean_list = np.expand_dims(np.array(mean_list), 0)
std_list = np.expand_dims(np.array(std_list), 0)
st_list = np.expand_dims(np.array(st_list), 0)
ed_list = np.expand_dims(np.array(ed_list), 0)
def get_scores_lin(prob1_list, prob2_list):
v2 = np.min(np.stack([prob1_list, prob2_list], axis=1), axis=1)
v2 = np.expand_dims(v2, 1)
all_scores = (v2 - mean_list) / std_list
prob1_list = np.expand_dims(prob1_list, 1)
f1 = np.less_equal(st_list, prob1_list)
f2 = np.less(prob1_list, ed_list)
f = np.logical_and(f1, f2)
all_scores = all_scores * f
scores = np.sum(all_scores, axis=1)
return scores
data = EstimatorPredictionViewerGosford(filename)
amp = 0.5
html_writer = HtmlVisualizer("{}_{}.html".format(name, amp), dark_mode=False)
for inst_i, entry in enumerate(data):
if inst_i > 10:
break
tokens = entry.get_mask_resolved_input_mask_with_input()
scores = entry.get_vector("priority_score")
loss1 = entry.get_vector("lm_loss1")
loss2 = entry.get_vector("lm_loss2")
#scores1 = get_scores_lin(loss_to_prob(loss1), loss_to_prob(loss2))
#scores = [get_score(v1, v2) for v1,v2 in zip(loss_to_prob(loss1), loss_to_prob(loss2))]
#assert np.all(np.less(np.abs(scores - scores1), 0.01))
prob_scores = probabilty(scores, amp)
prob_strs = ["{:06.6f}".format(v*1000) for v in prob_scores]
def normalize(prob):
# 0-> Good
# -1 -> Bad
return min(prob * 10000, 100)
norm_scores = lmap(normalize, prob_scores)
cells = data.cells_from_tokens(tokens, norm_scores)
cells2 = data.cells_from_anything(scores, norm_scores)
cells3 = data.cells_from_anything(prob_strs, norm_scores)
cells4 = data.cells_from_anything(loss_to_prob(loss1), norm_scores)
cells5 = data.cells_from_anything(loss_to_prob(loss2), norm_scores)
html_writer.multirow_print_from_cells_list([cells, cells2, cells3, cells4, cells5])
html_writer.write_headline("")
