def show(html_visualizer: HtmlVisualizer,
features: List[ParagraphClaimPersFeature]):
print("Cid: ", features[0].claim_pers.cid)
for f in features:
html_visualizer.write_paragraph("Claim: " + f.claim_pers.claim_text)
html_visualizer.write_paragraph("Perspective: " + f.claim_pers.p_text)
pc_tokens: List[str] = nltk.word_tokenize(
f.claim_pers.claim_text) + nltk.word_tokenize(f.claim_pers.p_text)
pc_tokens_set = set([t.lower() for t in pc_tokens])
print(pc_tokens_set)
def get_cell(token) -> Cell:
if token.lower() in pc_tokens_set:
score = 100
else:
score = 0
return Cell(token, score)
html_visualizer.write_paragraph("Label : {}".format(
f.claim_pers.label))
for score_paragraph in f.feature:
paragraph = score_paragraph.paragraph
cells = [get_cell(t) for t in paragraph.tokens]
html_visualizer.write_paragraph("---")
html_visualizer.multirow_print(cells, width=20)
def draw2(in_file, out_file):
filename = os.path.join(output_path, in_file)
data = EstimatorPredictionViewerGosford(filename)
html_writer = HtmlVisualizer(out_file, dark_mode=False)
tokenizer = get_tokenizer()
for inst_i, entry in enumerate(data):
if inst_i > 100:
break
tokens = entry.get_tokens("input_ids")
# tokens = entry.get_tokens("input_ids")
prob1 = entry.get_vector("prob1")
prob2 = entry.get_vector("prob2")
real_loss1 = entry.get_vector("per_example_loss1")
real_loss2 = entry.get_vector("per_example_loss2")
masked_lm_positions = entry.get_vector("masked_lm_positions")
for i, loc in enumerate(masked_lm_positions):
tokens[loc] = "[{}:{}]".format(i, tokens[loc])
html_writer.multirow_print(data.cells_from_tokens(tokens))
row2 = [Cell("prob1:")] + data.cells_from_anything(prob1)
row3 = [Cell("prob2:")] + data.cells_from_anything(prob2)
row4 = [Cell("real_loss1:")] + data.cells_from_anything(real_loss1)
row5 = [Cell("real_loss2:")] + data.cells_from_anything(real_loss2)
html_writer.multirow_print_from_cells_list([row2, row3, row4, row5])
def show(out_file_name, summarized_table: List[Entry]):
html = HtmlVisualizer(out_file_name)
tokenizer = get_tokenizer()
num_print = 0
for input_ids, prob, contributions in summarized_table:
tokens = tokenizer.convert_ids_to_tokens(input_ids)
html.write_paragraph("Score : {}".format(prob))
cells = []
max_change = 0
for idx in range(len(input_ids)):
token = tokens[idx]
if token == "[PAD]":
break
if idx in contributions:
raw_score = contributions[idx]
max_change = max(abs(raw_score), max_change)
score = abs(raw_score) * 100
color = "R" if raw_score > 0 else "B"
c = Cell(token, highlight_score=score, target_color=color)
else:
c = Cell(token, highlight_score=150, target_color="Gray")
cells.append(c)
if max_change < 0.05:
pass
else:
html.multirow_print(cells, 30)
num_print += 1
print("printed {} of {}".format(num_print, len(summarized_table)))
def run():
tokenizer = get_tokenizer()
spr = StreamPickleReader("contradiction_prediction")
html = HtmlVisualizer("contradiction_prediction.html")
cnt = 0
while spr.has_next():
item = spr.get_item()
e, p = item
input_ids, _, _ = e
logit, explain = p
tokens = tokenizer.convert_ids_to_tokens(input_ids)
p, h = split_p_h_with_input_ids(tokens, input_ids)
p_score, h_score = split_p_h_with_input_ids(explain, input_ids)
p_score = normalize(p_score)
h_score = normalize(h_score)
p_cells = [Cell("P:")] + cells_from_tokens(p, p_score)
h_cells = [Cell("H:")] + cells_from_tokens(h, h_score)
html.write_paragraph(str(logit))
html.multirow_print(p_cells)
html.multirow_print(h_cells)
if cnt > 100:
break
cnt += 1
def view_grad_overlap_per_mask():
filename = "ukp_lm_probs.pickle"
out_name = filename.split(".")[0] + ".html"
html_writer = HtmlVisualizer(out_name, dark_mode=False)
data = EstimatorPredictionViewerGosford(filename)
tokenizer = data.tokenizer
for inst_i, entry in enumerate(data):
tokens = entry.get_mask_resolved_input_mask_with_input()
highlight = lmap(is_mask, tokens)
scores = entry.get_vector("overlap_score")
pos_list = entry.get_vector("masked_lm_positions")
probs = entry.get_vector("masked_lm_log_probs")
probs = np.reshape(probs, [20, -1])
rows = []
for score, position, prob in zip(scores, pos_list, probs):
tokens[position] = "{}-".format(position) + tokens[position]
row = [Cell(position), Cell(score)]
for idx in np.argsort(prob)[::-1][:5]:
term = tokenizer.inv_vocab[idx]
p = math.exp(prob[idx])
row.append(Cell(term))
row.append(Cell(p))
rows.append(row)
cells = data.cells_from_tokens(tokens, highlight)
for score, position in zip(scores, pos_list):
cells[position].highlight_score = score / 10000 * 255
html_writer.multirow_print(cells, 20)
html_writer.write_table(rows)
def draw():
#name = "pc_para_D_grad"
name = "pc_para_I_grad"
#name = "pc_para_H_grad"
data = EstimatorPredictionViewerGosford(name)
html_writer = HtmlVisualizer(name + ".html", dark_mode=False)
for inst_i, entry in enumerate(data):
tokens = entry.get_tokens("input_ids")
grad = entry.get_vector("gradient")
m = min(grad)
cells = data.cells_from_tokens(tokens)
for i, cell in enumerate(cells):
cells[i].highlight_score = min(abs(grad[i]) * 1e4, 255)
cells[i].target_color = "B" if grad[i] > 0 else "R"
print(grad)
prob = softmax(entry.get_vector("logits"))
pred = np.argmax(prob)
label = entry.get_vector("labels")
html_writer.write_paragraph("Label={} / Pred={}".format(str(label), pred))
html_writer.multirow_print(cells)
def view_grad_overlap():
filename = "gradient_overlap_4K.pickle"
out_name = filename.split(".")[0] + ".html"
html_writer = HtmlVisualizer(out_name, dark_mode=False)
data = EstimatorPredictionViewerGosford(filename)
iba = IntBinAverage()
scores = []
for inst_i, entry in enumerate(data):
masked_lm_example_loss = entry.get_vector("masked_lm_example_loss")
score = entry.get_vector("overlap_score")
if masked_lm_example_loss > 1:
norm_score = score / masked_lm_example_loss
iba.add(masked_lm_example_loss, norm_score)
scores.append(score)
score_avg = average(scores)
score_std = np.std(scores)
avg = iba.all_average()
std_dict = {}
for key, values in iba.list_dict.items():
std_dict[key] = np.std(values)
if len(values) == 1:
std_dict[key] = 999
def unlikeliness(value, mean, std):
return abs(value - mean) / std
data = EstimatorPredictionViewerGosford(filename)
print("num record : ", data.data_len)
cnt = 0
for inst_i, entry in enumerate(data):
tokens = entry.get_mask_resolved_input_mask_with_input()
masked_lm_example_loss = entry.get_vector("masked_lm_example_loss")
highlight = lmap(is_mask, tokens)
score = entry.get_vector("overlap_score")
print(score)
cells = data.cells_from_tokens(tokens, highlight)
if masked_lm_example_loss > 1:
bin_key = int(masked_lm_example_loss)
norm_score = score / masked_lm_example_loss
if norm_score > 5000:
cnt += 1
expectation = avg[bin_key]
if unlikeliness(score, score_avg, score_std) > 2 or True:
html_writer.multirow_print(cells, 20)
if norm_score > expectation:
html_writer.write_paragraph("High")
else:
html_writer.write_paragraph("Low")
html_writer.write_paragraph("Norm score: " + str(norm_score))
html_writer.write_paragraph("score: " + str(score))
html_writer.write_paragraph("masked_lm_example_loss: " +
str(masked_lm_example_loss))
html_writer.write_paragraph("expectation: " + str(expectation))
print("number over 5000: ", cnt)
def loss_view(dir_path):
tokenizer = get_tokenizer()
html_writer = HtmlVisualizer("ukp_lm_grad_high.html", dark_mode=False)
for file_path in get_dir_files(dir_path):
items = pickle.load(open(file_path, "rb"))
for e in items:
input_ids, masked_input_ids, masked_lm_example_loss = e
tokens = mask_resolve_1(
tokenizer.convert_ids_to_tokens(input_ids),
tokenizer.convert_ids_to_tokens(masked_input_ids))
highlight = lmap(is_mask, tokens)
cells = cells_from_tokens(tokens, highlight)
html_writer.multirow_print(cells)
def print_as_html(fn):
examples = load_record(fn)
tokenizer = tokenizer_wo_tf.FullTokenizer(
os.path.join(data_path, "bert_voca.txt"))
html_output = HtmlVisualizer("out_name.html")
for feature in examples:
masked_inputs = feature["input_ids"].int64_list.value
idx = 0
step = 512
while idx < len(masked_inputs):
slice = masked_inputs[idx:idx + step]
tokens = tokenizer.convert_ids_to_tokens(slice)
idx += step
cells = cells_from_tokens(tokens)
html_output.multirow_print(cells)
html_output.write_paragraph("----------")
def loss_view():
filename = "sero_pred.pickle"
p = os.path.join(output_path, filename)
data = pickle.load(open(p, "rb"))
print(data[0]["masked_lm_example_loss"].shape)
print(data[0]["masked_input_ids"].shape)
html_writer = HtmlVisualizer("sero_pred.html", dark_mode=False)
data = EstimatorPredictionViewerGosford(filename)
for inst_i, entry in enumerate(data):
losses = entry.get_vector("masked_lm_example_loss")
print(losses)
tokens = entry.get_tokens("masked_input_ids")
cells = data.cells_from_tokens(tokens)
row = []
for idx, cell in enumerate(cells):
row.append(cell)
if len(row) == 20:
html_writer.write_table([row])
row = []
html_writer.multirow_print(data.cells_from_anything(losses), 20)
def main():
n_factor = 16
step_size = 16
max_seq_length = 128
max_seq_length2 = 128 - 16
batch_size = 8
info_file_path = at_output_dir("robust", "seg_info")
queries = load_robust_04_query("desc")
qid_list = get_robust_qid_list()
f_handler = get_format_handler("qc")
info: Dict = load_combine_info_jsons(info_file_path,
f_handler.get_mapping(),
f_handler.drop_kdp())
print(len(info))
tokenizer = get_tokenizer()
for job_idx in [1]:
qid = qid_list[job_idx]
query = queries[str(qid)]
q_term_length = len(tokenizer.tokenize(query))
data_path1 = os.path.join(output_path, "robust",
"windowed_{}.score".format(job_idx))
data_path2 = os.path.join(output_path, "robust",
"windowed_small_{}.score".format(job_idx))
data1 = OutputViewer(data_path1, n_factor, batch_size)
data2 = OutputViewer(data_path2, n_factor, batch_size)
segment_len = max_seq_length - 3 - q_term_length
segment_len2 = max_seq_length2 - 3 - q_term_length
outputs = []
for d1, d2 in zip(data1, data2):
# for each query, doc pairs
cur_info1 = info[d1['data_id']]
cur_info2 = info[d2['data_id']]
query_doc_id1 = f_handler.get_pair_id(cur_info1)
query_doc_id2 = f_handler.get_pair_id(cur_info2)
assert query_doc_id1 == query_doc_id2
doc = d1['doc']
probs = get_probs(d1['logits'])
probs2 = get_probs(d2['logits'])
n_pred_true = np.count_nonzero(np.less(0.5, probs))
print(n_pred_true, len(probs))
seg_scores: List[Tuple[int, int, float]] = get_piece_scores(
n_factor, probs, segment_len, step_size)
seg_scores2: List[Tuple[int, int, float]] = get_piece_scores(
n_factor, probs2, segment_len2, step_size)
ss_list = []
for st, ed, score in seg_scores:
try:
st2, ed2, score2 = find_where(lambda x: x[1] == ed,
seg_scores2)
assert ed == ed2
assert st < st2
tokens = tokenizer.convert_ids_to_tokens(doc[st:st2])
diff = score - score2
ss = ScoredPiece(st, st2, diff, tokens)
ss_list.append(ss)
except StopIteration:
pass
outputs.append((probs, probs2, query_doc_id1, ss_list))
html = HtmlVisualizer("windowed.html")
for probs, probs2, query_doc_id, ss_list in outputs:
html.write_paragraph(str(query_doc_id))
html.write_paragraph("Query: " + query)
ss_list.sort(key=lambda ss: ss.st)
prev_end = None
cells = []
prob_str1 = lmap(two_digit_float, probs)
prob_str1 = ["8.88"] + prob_str1
prob_str2 = lmap(two_digit_float, probs2)
html.write_paragraph(" ".join(prob_str1))
html.write_paragraph(" ".join(prob_str2))
for ss in ss_list:
if prev_end is not None:
assert prev_end == ss.st
else:
print(ss.st)
score = abs(int(100 * ss.score))
color = "B" if score > 0 else "R"
cells.extend(
[Cell(t, score, target_color=color) for t in ss.tokens])
prev_end = ss.ed
html.multirow_print(cells)
def binary_feature_demo(datapoint_list):
ci = PassageRankedListInterface(make_passage_query, Q_CONFIG_ID_BM25)
not_found_set = set()
_, clue12_13_df = load_clueweb12_B13_termstat()
cdf = 50 * 1000 * 1000
html = HtmlVisualizer("pc_binary_feature.html")
def idf_scorer(doc, claim_text, perspective_text):
cp_tokens = nltk.word_tokenize(claim_text) + nltk.word_tokenize(
perspective_text)
cp_tokens = lmap(lambda x: x.lower(), cp_tokens)
cp_tokens = set(cp_tokens)
mentioned_terms = lfilter(lambda x: x in doc, cp_tokens)
mentioned_terms = re_tokenize(mentioned_terms)
def idf(term):
if term not in clue12_13_df:
if term in string.printable:
return 0
not_found_set.add(term)
return math.log((cdf + 0.5) / (clue12_13_df[term] + 0.5))
score = sum(lmap(idf, mentioned_terms))
max_score = sum(lmap(idf, cp_tokens))
# print(claim_text, perspective_text)
# print(mentioned_terms)
# print(score, max_score)
return score, max_score, mentioned_terms
def bm25_estimator(doc: Counter, claim_text: str, perspective_text: str):
cp_tokens = nltk.word_tokenize(claim_text) + nltk.word_tokenize(
perspective_text)
cp_tokens = lmap(lambda x: x.lower(), cp_tokens)
k1 = 0
def BM25_3(f, qf, df, N, dl, avdl) -> float:
K = compute_K(dl, avdl)
first = math.log((N - df + 0.5) / (df + 0.5))
second = ((k1 + 1) * f) / (K + f)
return first * second
dl = sum(doc.values())
info = []
for q_term in set(cp_tokens):
if q_term in doc:
score = BM25_3(doc[q_term], 0, clue12_13_df[q_term], cdf, dl,
1200)
info.append((q_term, doc[q_term], clue12_13_df[q_term], score))
return info
print_cnt = 0
for dp_idx, x in enumerate(datapoint_list):
ranked_list: List[GalagoRankEntry] = ci.query_passage(
x.cid, x.pid, x.claim_text, x.p_text)
html.write_paragraph(x.claim_text)
html.write_paragraph(x.p_text)
html.write_paragraph("{}".format(x.label))
local_print_cnt = 0
lines = []
for ranked_entry in ranked_list:
try:
doc_id = ranked_entry.doc_id
galago_score = ranked_entry.score
tokens = load_doc(doc_id)
doc_tf = Counter(tokens)
if doc_tf is not None:
score, max_score, mentioned_terms = idf_scorer(
doc_tf, x.claim_text, x.p_text)
matched = score > max_score * 0.75
else:
matched = "Unk"
score = "Unk"
max_score = "Unk"
def get_cell(token):
if token in mentioned_terms:
return Cell(token, highlight_score=50)
else:
return Cell(token)
line = [doc_id, galago_score, matched, score, max_score]
lines.append(line)
html.write_paragraph("{0} / {1:.2f}".format(
doc_id, galago_score))
html.write_paragraph("{}/{}".format(score, max_score))
bm25_info = bm25_estimator(doc_tf, x.claim_text, x.p_text)
bm25_score = sum(lmap(lambda x: x[3], bm25_info))
html.write_paragraph(
"bm25 re-estimate : {}".format(bm25_score))
html.write_paragraph("{}".format(bm25_info))
html.multirow_print(lmap(get_cell, tokens))
local_print_cnt += 1
if local_print_cnt > 10:
break
except KeyError:
pass
matched_idx = idx_where(lambda x: x[2], lines)
if not matched_idx:
html.write_paragraph("No match")
else:
last_matched = matched_idx[-1]
lines = lines[:last_matched + 1]
rows = lmap(lambda line: lmap(Cell, line), lines)
html.write_table(rows)
if dp_idx > 10:
break
def main():
prediction_file_path = at_output_dir("robust", "rob_dense_pred.score")
info_file_path = at_job_man_dir1("robust_predict_desc_128_step16_info")
queries: Dict[str, str] = load_robust_04_query("desc")
tokenizer = get_tokenizer()
query_token_len_d = {}
for qid, q_text in queries.items():
query_token_len_d[qid] = len(tokenizer.tokenize(q_text))
step_size = 16
window_size = 128
out_entries: List[DocTokenScore] = collect_token_scores(
info_file_path, prediction_file_path, query_token_len_d, step_size,
window_size)
qrel_path = "/home/youngwookim/Downloads/rob04-desc/qrels.rob04.txt"
judgement_d = load_qrels_structured(qrel_path)
html = HtmlVisualizer("robust_desc_128_step16.html", use_tooltip=True)
tprint("loading tokens pickles")
tokens_d: Dict[str, List[str]] = load_pickle_from(
os.path.join(sydney_working_dir, "RobustPredictTokens3", "1"))
tprint("Now printing")
n_printed = 0
def transform(x):
return 3 * (math.pow(x - 0.5, 3) + math.pow(0.5, 3))
for e in out_entries:
max_score = e.max_segment_score()
if max_score < 0.6:
continue
n_printed += 1
if n_printed > 10:
break
doc_tokens: List[str] = tokens_d[e.doc_id]
score_len = len(e.scores)
judgement: Dict[str, int] = judgement_d[e.query_id]
label = judgement[e.doc_id]
if not len(doc_tokens) <= score_len < len(doc_tokens) + window_size:
print("doc length : ", len(doc_tokens))
print("score len:", score_len)
print("doc length +step_size: ", len(doc_tokens) + step_size)
raise IndexError
row = []
q_text = queries[e.query_id]
html.write_paragraph("qid: " + e.query_id)
html.write_paragraph("q_text: " + q_text)
html.write_paragraph("Pred: {0:.2f}".format(max_score))
html.write_paragraph("Label: {0:.2f}".format(label))
for idx in range(score_len):
token = doc_tokens[idx] if idx < len(doc_tokens) else '[-]'
full_scores = e.full_scores[idx]
full_score_str = " ".join(lmap(two_digit_float, full_scores))
score = e.scores[idx]
normalized_score = transform(score) * 200
c = get_tooltip_cell(token, full_score_str)
c.highlight_score = normalized_score
row.append(c)
html.multirow_print(row, 16)
def join_docs_and_lm():
gold = get_claim_perspective_id_dict()
d_ids = list(load_train_claim_ids())
claims: List[Dict] = get_claims_from_ids(d_ids)
claims = claims[:10]
top_n = 10
q_res_path = FilePath(
"/mnt/nfs/work3/youngwookim/data/perspective/train_claim/q_res_100")
ranked_list: Dict[
str, List[SimpleRankedListEntry]] = load_galago_ranked_list(q_res_path)
preload_docs(ranked_list, claims, top_n)
claim_lms = build_gold_lms(claims)
claim_lms_d = {lm.cid: lm for lm in claim_lms}
bg_lm = average_counters(lmap(lambda x: x.LM, claim_lms))
log_bg_lm = get_lm_log(bg_lm)
stopwords.update([".", ",", "!", "?"])
alpha = 0.1
html_visualizer = HtmlVisualizer("doc_lm_joined.html")
def get_cell_from_token2(token, probs):
if token.lower() in stopwords:
probs = 0
probs = probs * 1e5
s = min(100, probs)
c = Cell(token, s)
return c
tokenizer = PCTokenizer()
for c in claims:
q_res: List[SimpleRankedListEntry] = ranked_list[str(c['cId'])]
html_visualizer.write_headline("{} : {}".format(c['cId'], c['text']))
clusters: List[List[int]] = gold[c['cId']]
for cluster in clusters:
html_visualizer.write_paragraph("---")
p_text_list: List[str] = lmap(perspective_getter, cluster)
for text in p_text_list:
html_visualizer.write_paragraph(text)
html_visualizer.write_paragraph("---")
claim_lm = claim_lms_d[c['cId']]
topic_lm_prob = smooth(claim_lm.LM, bg_lm, alpha)
log_topic_lm = get_lm_log(smooth(claim_lm.LM, bg_lm, alpha))
log_odd: Counter = subtract(log_topic_lm, log_bg_lm)
s = "\t".join(left(log_odd.most_common(30)))
html_visualizer.write_paragraph("Log odd top: " + s)
not_found = set()
def get_log_odd(x):
x = tokenizer.stemmer.stem(x)
if x not in log_odd:
not_found.add(x)
return log_odd[x]
def get_probs(x):
x = tokenizer.stemmer.stem(x)
if x not in topic_lm_prob:
not_found.add(x)
return topic_lm_prob[x]
for i in range(top_n):
try:
doc = load_doc(q_res[i].doc_id)
cells = lmap(lambda x: get_cell_from_token(x, get_log_odd(x)),
doc)
html_visualizer.write_headline("Doc rank {}".format(i))
html_visualizer.multirow_print(cells, width=20)
except KeyError:
pass
html_visualizer.write_paragraph("Not found: {}".format(not_found))
def main():
cid = sys.argv[1]
print("Loading scores...")
baseline_cid_grouped = load_baseline("train_baseline")
gold = get_claim_perspective_id_dict()
tokenizer = get_tokenizer()
claim_d = load_train_claim_d()
print("Start analyzing")
html = HtmlVisualizer("cppnc_value_per_token_score_{}.html".format(cid))
pid_entries_d = load_entries(cid)
pid_entries_d: Dict[str, List[Dict]] = pid_entries_d
pid_entries: List[Tuple[str, List[Dict]]] = list(pid_entries_d.items())
baseline_pid_entries = baseline_cid_grouped[int(cid)]
baseline_score_d = fetch_score_per_pid(baseline_pid_entries)
gold_pids = gold[int(cid)]
ret = collect_score_per_doc(baseline_score_d, get_score_from_entry, gold_pids,
pid_entries)
passage_tokens_d = collect_passage_tokens(pid_entries)
doc_info_d: Dict[int, Tuple[str, int]] = ret[0]
doc_value_arr: List[List[float]] = ret[1]
kdp_result_grouped = defaultdict(list)
for doc_idx, doc_values in enumerate(doc_value_arr):
doc_id, passage_idx = doc_info_d[doc_idx]
avg_score = average(doc_values)
kdp_result = doc_id, passage_idx, avg_score
kdp_result_grouped[doc_id].append(kdp_result)
s = "{} : {}".format(cid, claim_d[int(cid)])
html.write_headline(s)
scores: List[float] = list([r[2] for r in doc_value_arr])
foreach(html.write_paragraph, lmap(str, scores))
for doc_id, kdp_result_list in kdp_result_grouped.items():
html.write_headline(doc_id)
tokens, per_token_score = combine_collect_score(tokenizer, doc_id, passage_tokens_d, kdp_result_list)
str_tokens = tokenizer.convert_ids_to_tokens(tokens)
row = cells_from_tokens(str_tokens)
for idx in range(len(str_tokens)):
score = average(per_token_score[idx])
norm_score = min(abs(score) * 10000, 100)
color = "B" if score > 0 else "R"
row[idx].highlight_score = norm_score
row[idx].target_color = color
rows = [row]
nth = 0
any_score_found = True
while any_score_found:
any_score_found = False
score_list = []
for idx in range(len(str_tokens)):
if nth < len(per_token_score[idx]):
score = per_token_score[idx][nth]
any_score_found = True
else:
score = "-"
score_list.append(score)
def get_cell(score):
if score == "-":
return Cell("-")
else:
# 0.01 -> 100
norm_score = min(abs(score) * 20000, 255)
color = "B" if score > 0 else "R"
return Cell("", highlight_score=norm_score, target_color=color)
nth += 1
if any_score_found:
row = lmap(get_cell, score_list)
rows.append(row)
html.multirow_print(rows[0])
def main():
prediction_file_path = at_output_dir("robust", "rob_dense2_pred.score")
info_file_path = at_job_man_dir1("robust_predict_desc_128_step16_2_info")
queries: Dict[str, str] = load_robust_04_query("desc")
tokenizer = get_tokenizer()
query_token_len_d = {}
for qid, q_text in queries.items():
query_token_len_d[qid] = len(tokenizer.tokenize(q_text))
step_size = 16
window_size = 128
out_entries: List[AnalyzedDoc] = token_score_by_ablation(
info_file_path, prediction_file_path, query_token_len_d, step_size,
window_size)
qrel_path = "/home/youngwookim/Downloads/rob04-desc/qrels.rob04.txt"
judgement_d = load_qrels_structured(qrel_path)
html = HtmlVisualizer("robust_desc_128_step16_2.html", use_tooltip=True)
tprint("loading tokens pickles")
tokens_d: Dict[str, List[str]] = load_pickle_from(
os.path.join(sydney_working_dir, "RobustPredictTokens3", "1"))
tprint("Now printing")
n_printed = 0
def transform(x):
return 3 * (math.pow(x - 0.5, 3) + math.pow(0.5, 3))
n_pos = 0
n_neg = 0
for e in out_entries:
max_score: float = max(
lmap(SegmentScorePair.get_max_score,
flatten(e.token_info.values())))
if max_score < 0.6:
if n_neg > n_pos:
continue
else:
n_neg += 1
pass
else:
n_pos += 1
n_printed += 1
if n_printed > 500:
break
doc_tokens: List[str] = tokens_d[e.doc_id]
score_len = max(e.token_info.keys()) + 1
judgement: Dict[str, int] = judgement_d[e.query_id]
label = judgement[e.doc_id]
if not len(doc_tokens) <= score_len < len(doc_tokens) + window_size:
print("doc length : ", len(doc_tokens))
print("score len:", score_len)
print("doc length +step_size: ", len(doc_tokens) + step_size)
continue
row = []
q_text = queries[e.query_id]
html.write_paragraph("qid: " + e.query_id)
html.write_paragraph("q_text: " + q_text)
html.write_paragraph("Pred: {0:.2f}".format(max_score))
html.write_paragraph("Label: {0:.2f}".format(label))
for idx in range(score_len):
token = doc_tokens[idx] if idx < len(doc_tokens) else '[-]'
token_info: List[SegmentScorePair] = e.token_info[idx]
full_scores: List[float] = lmap(SegmentScorePair.get_score_diff,
token_info)
full_score_str = " ".join(lmap(two_digit_float, full_scores))
# 1 ~ -1
score = average(full_scores)
if score > 0:
color = "B"
else:
color = "R"
normalized_score = transform(abs(score)) * 200
c = get_tooltip_cell(token, full_score_str)
c.highlight_score = normalized_score
c.target_color = color
row.append(c)
html.multirow_print(row, 16)
