def get_doc_value_parts2(out_entries: List[QCKOutEntry],
baseline_score_d: Dict[Tuple[str, str], float],
gold: Dict[str, List[str]]) -> List[DocValueParts2]:
def get_qid(entry: QCKOutEntry):
return entry.query.query_id
def get_candidate_id(entry: QCKOutEntry):
return entry.candidate.id
print("baseline has {} entries".format(len(baseline_score_d.keys())))
print("baseline has {} qids".format(
len(group_by(baseline_score_d.keys(), get_first))))
not_found_baseline = set()
dvp_entries = []
for qid, entries in group_by(out_entries, get_qid).items():
gold_candidate_ids: List[str] = gold[qid]
candi_grouped = group_by(entries, get_candidate_id)
print(qid, len(candi_grouped))
for candidate_id, sub_entries in candi_grouped.items():
try:
key = qid, candidate_id
base_score: float = baseline_score_d[key]
label = candidate_id in gold_candidate_ids
sub_entries: List[QCKOutEntry] = sub_entries
for e in sub_entries:
dvp = DocValueParts2.init(e, label, base_score)
dvp_entries.append(dvp)
except KeyError:
not_found_baseline.add(key)
if len(not_found_baseline) > 32:
print(not_found_baseline)
raise KeyError()
return dvp_entries
def group_by_qid_cid(
predictions: List[Dict]) -> Dict[str, Dict[str, List[Dict]]]:
grouped: Dict[str, List[Dict]] = group_by(predictions,
lambda x: x['query'].query_id)
grouped2: Dict[str, Dict[str, List[Dict]]] = \
dict_value_map(lambda x: group_by(x, lambda x: x['candidate'].id), grouped)
return grouped2
def main():
dvp: List[DocValueParts] = load()
def get_qid(e: DocValueParts):
return e.query.query_id
# Group by doc id
dvp_qid_grouped: Dict[str, List[DocValueParts]] = group_by(dvp, get_qid)
def get_doc_id_idx(e: DocValueParts):
return e.kdp.doc_id, e.kdp.passage_idx
def get_doc_id(e: DocValueParts):
return e.kdp.doc_id
value_types = ["good", "bad", "none"]
head = ["qid"] + value_types
rows = [head]
rows2 = []
for qid, entries in dvp_qid_grouped.items():
# Q : How many kdp are useful?
# Q : Does relevant matter?
kdp_grouped = group_by(entries, get_doc_id_idx)
counter = Counter()
doc_value = Counter()
for kdp_id, entries2 in kdp_grouped.items():
doc_id, _ = kdp_id
value_avg: float = sum([e.value for e in entries2])
if value_avg > 1:
counter["good"] += 1
doc_value[doc_id] += 1
elif value_avg < -1:
counter["bad"] += 1
doc_value[doc_id] -= 1
else:
counter["none"] += 1
row = [qid] + [counter[k] for k in value_types]
rows.append(row)
doc_value_list = list(doc_value.items())
doc_value_list.sort(key=get_second, reverse=True)
rows2.append([qid])
for doc_id, value in doc_value_list[:10]:
if value > 0:
rows2.append([doc_id, value])
doc_value_list.sort(key=get_second)
for doc_id, value in doc_value_list[:10]:
if value < 0:
rows2.append([doc_id, value])
print_table(rows)
print_table(rows2)
def sanity_check():
dvp: List[DocValueParts2] = load()
candidate_d_raw: List[Tuple[int, List[int]]] = get_eval_candidate_as_pids(
"train")
candidate_d = {str(k): lmap(str, v) for k, v in candidate_d_raw}
# Group by doc id
dvp_qid_grouped: Dict[str, List[DocValueParts2]] = group_by(dvp, get_qid)
ap_baseline = []
ap_new_score = []
for qid, entries in dvp_qid_grouped.items():
ranked_list_new = []
ranked_list_baseline = []
candidate_id_grouped = group_by(entries, get_candidate)
for candidate_id, entries2 in candidate_id_grouped.items():
is_initial_candidate = candidate_id in candidate_d[qid]
gold = entries2[0].label
skip = gold and not is_initial_candidate
def get_new_score(dvp: DocValueParts2):
return dvp.score
def get_baseline_score(dvp: DocValueParts2):
return dvp.init_score
if skip:
continue
new_score = top_k_avg(lmap(get_new_score, entries2))
baseline_score = average(lmap(get_baseline_score, entries2))
ranked_list_new.append((candidate_id, new_score, gold))
ranked_list_baseline.append((candidate_id, baseline_score, gold))
def get_ap(ranked_list):
ranked_list.sort(key=lambda x: x[1], reverse=True)
p_list = []
p = 0
for rank, (cid, score, gold) in enumerate(ranked_list):
if gold:
p += 1
p_list.append(p / (rank + 1))
return average(p_list)
ap_baseline.append(get_ap(ranked_list_baseline))
ap_new_score.append(get_ap(ranked_list_new))
print("MAP baseline", average(ap_baseline))
print("MAP new score", average(ap_new_score))
def get_confidence_list_per_cid(info_dir, prediction_file) -> Dict[int, List[float]]:
info = load_combine_info_jsons(info_dir)
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
scores: Dict[DataID, Tuple[CPIDPair, float, float]] = collect_scores_and_confidence(prediction_file, info, logit_to_score_softmax)
grouped = group_by(scores.values(), lambda x: x[0])
print("Group size:", len(grouped))
entries = group_by_cpid(grouped)
cid_grouped = group_by(entries, lambda x: x[0])
verify_confidence_consistency(cid_grouped)
return dict_value_map(lambda x: x[0][2], cid_grouped)
def load_prediction(
data: EstimatorPredictionViewer) -> List[Tuple[str, List[float]]]:
print("prediction has {} entry".format(data.data_len))
def parse_entry(entry) -> Tuple[str, float]:
input_tokens: Segment = entry.get_tokens('input_ids')
logits = entry.get_vector("logits")
probs = softmax(logits)
key = input_tokens_to_key(input_tokens)
score = probs[1]
return key, score
parsed_data: List[Tuple[str, float]] = lmap(parse_entry, data)
keys: List[str] = unique_from_sorted(left(parsed_data))
grouped: Dict[str, List[Tuple[str,
float]]] = group_by(parsed_data,
lambda x: x[0])
def fetch_scores(key):
l = []
for k2, score in grouped[key]:
assert key == k2
l.append(score)
return key, l
results: List[Tuple[str, List[float]]] = lmap(fetch_scores, keys)
return results
def load_ranked_list_grouped(path) -> Dict[str, List[TrecRankedListEntry]]:
ranked_list: List[TrecRankedListEntry] = load_ranked_list(path)
def get_qid(e: TrecRankedListEntry):
return e.query_id
return group_by(ranked_list, get_qid)
def summarize_score(info: Dict, prediction_file_path: str,
f_handler: FormatHandler, combine_score: Callable,
score_type) -> Dict[Tuple[str, str], float]:
key_logit = "logits"
data: List[Dict] = join_prediction_with_info(prediction_file_path, info,
["data_id", key_logit])
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
def get_score(entry):
if score_type == "softmax":
return logit_to_score_softmax(entry['logits'])
elif score_type == "raw":
return entry[key_logit][0]
elif score_type == "scalar":
return entry[key_logit]
elif score_type == "tuple":
return entry[key_logit][1]
else:
assert False
grouped: Dict[Tuple[str, str],
List[Dict]] = group_by(data, f_handler.get_pair_id)
tprint("Group size:", len(grouped))
out_d = {}
for pair_id, items in grouped.items():
scores = lmap(get_score, items)
final_score = combine_score(scores)
out_d[pair_id] = final_score
num_items_per_group = average(lmap(len, grouped.values()))
tprint("Num items per group : ", num_items_per_group)
return out_d
def summarize_score_wo_merge(info: Dict, prediction_file_path: str,
f_handler: FormatHandler,
score_type) -> Dict[Tuple[str, str], float]:
key_logit = "logits"
data: List[Dict] = join_prediction_with_info(prediction_file_path, info,
["data_id", key_logit])
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
def get_score(entry):
if score_type == "softmax":
return logit_to_score_softmax(entry['logits'])
elif score_type == "raw":
return entry[key_logit][0]
elif score_type == "scalar":
return entry[key_logit]
elif score_type == "tuple":
return entry[key_logit][1]
else:
assert False
grouped: Dict[Tuple[str, str],
List[Dict]] = group_by(data, f_handler.get_pair_id)
tprint("Group size:", len(grouped))
out_d = {}
for pair_id, items in grouped.items():
query_id, doc_id = pair_id
scores = lmap(get_score, items)
for idx, score in enumerate(scores):
new_doc_id = "{}_{}".format(doc_id, idx)
out_d[(query_id, new_doc_id)] = score
return out_d
def load_prediction(pred_path) -> List[Tuple[str, List[np.ndarray]]]:
data = EstimatorPredictionViewer(pred_path)
def parse_entry(entry) -> Tuple[str, np.ndarray]:
input_tokens: Segment = entry.get_tokens('input_ids')
logits = entry.get_vector("logits")
probs = softmax(logits)
key = input_tokens_to_key(input_tokens)
return key, probs
parsed_data: List[Tuple[str, np.ndarray]] = lmap(parse_entry, data)
keys: List[str] = unique_from_sorted(left(parsed_data))
grouped: Dict[str,
List[Tuple[str,
np.ndarray]]] = group_by(parsed_data,
lambda x: x[0])
def fetch_scores(key):
l = []
for k2, score in grouped[key]:
assert key == k2
l.append(score)
return key, l
results: List[Tuple[str, List[np.ndarray]]] = lmap(fetch_scores, keys)
return results
def group_average_per_query(outputs) -> Dict[str, Dict[WordAsID, np.array]]:
tokenizer = get_tokenizer()
def collect_by_word_fn(entry: QKTokenLevelOutEntry):
return collect_by_words(tokenizer, entry)
print("Grouping entries")
grouped: Dict[str, List[QKTokenLevelOutEntry]] = group_by(outputs, lambda x: x.query.query_id)
def average_scores(out_entries: List[QKTokenLevelOutEntry]) -> Dict[WordAsID, np.array]:
items: List[Iterable[Tuple[WordAsID, TokenScore]]] = lmap(collect_by_word_fn, out_entries)
d: Dict[WordAsID, List] = defaultdict(list)
for item in items:
item: Iterable[Tuple[WordAsID, TokenScore]] = item
for word, probs in item:
d[word].append(probs)
def average_per_dim(probs_list) -> np.array:
return np.mean(np.array(probs_list), axis=0)
out_d: Dict[WordAsID, np.array] = dict_value_map(average_per_dim, d)
return out_d
print("Collecting token level scores")
per_query_infos: Dict[str, Dict[WordAsID, np.array]] = {}
ticker = TimeEstimator(len(grouped))
for key, value in grouped.items():
per_query_infos[key] = average_scores(value)
ticker.tick()
return per_query_infos
def summarize_score(info: Dict, prediction_file_path: str,
f_handler: FormatHandler,
score_type) -> Iterable[TrecRankedListEntry]:
key_logit = "logits"
data: List[Dict] = join_prediction_with_info(prediction_file_path, info,
["data_id", key_logit])
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
def get_score(entry):
if score_type == "softmax":
return logit_to_score_softmax(entry['logits'])
elif score_type == "raw":
return entry[key_logit][0]
elif score_type == "scalar":
return entry[key_logit]
elif score_type == "tuple":
return entry[key_logit][1]
else:
assert False
grouped: Dict[Tuple[str, str],
List[Dict]] = group_by(data, f_handler.get_pair_id)
print("Group size:", len(grouped))
out_d = {}
for pair_id, items in grouped.items():
scores = lmap(get_score, items)
query_id, doc_id = pair_id
out_d[pair_id] = scores
for score in scores:
yield TrecRankedListEntry(query_id, doc_id, 0, score, "")
def summarize_score(info_dir, prediction_file) -> Dict[CPIDPair, float]:
info = load_combine_info_jsons(info_dir)
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
scores: Dict[DataID, Tuple[CPIDPair, float,
float]] = collect_scores_and_confidence(
prediction_file, info,
logit_to_score_softmax)
grouped = group_by(scores.values(), lambda x: x[0])
print("Group size:", len(grouped))
out_d = {}
for cpid, items in grouped.items():
a = 0
b = 0
score_list = list([e[1] for e in items])
conf_list = list([e[2] for e in items])
print(cpid)
print(lmap("{0:.2f}".format, score_list))
print(lmap("{0:.2f}".format, conf_list))
for cpid, score, confidence in items:
a += score * confidence
b += confidence
final_score = a / (b + 0.0001)
out_d[cpid] = final_score
return out_d
def group_and_extract_ngram(subword_convertor: SubwordConvertor, max_para,
entries) -> List[PCNGramFeature]:
n_list = [1, 2, 3]
all_n_grams = {n: Counter() for n in n_list}
# output_entry = cpid, label, paragraph, c_score, p_score
def get_cpid(entry) -> CPIDPair:
return entry[0]
def get_paragraph(entry):
return entry[2]
def get_label(entry):
return entry[1]
grouped = group_by(entries, get_cpid)
r = []
for cpid, entries in grouped.items():
for e in entries[:max_para]:
word_list = subword_convertor.get_words(get_paragraph(e))
for n in n_list:
ngram_list = nltk.ngrams(word_list, n)
all_n_grams[n].update(ngram_list)
cid, pid = cpid
claim_pers = PerspectiveCandidate(label=str(get_label(entries[0])),
cid=cid,
pid=pid,
claim_text="",
p_text="")
pc_ngram_feature = PCNGramFeature(claim_pers=claim_pers,
n_grams=all_n_grams)
r.append(pc_ngram_feature)
return r
def extract_qk_unit(info_path, pred_path, config_path) -> Iterable[QKUnit]:
info = load_combine_info_jsons(info_path, qk_convert_map, False)
predictions = join_prediction_with_info(pred_path, info)
grouped: Dict[str, List[Dict]] = group_by(predictions,
lambda x: x['query'].query_id)
config = json.load(open(config_path, "r"))
score_cut = config['score_cut']
top_k = config['top_k']
def is_good(entry):
return get_regression_score(entry) > score_cut
select_rate_list = []
qk_units = []
for qid, entries in grouped.items():
any_entry = entries[0]
query = any_entry['query']
good_entries = lfilter(is_good, entries)
good_entries.sort(key=get_regression_score, reverse=True)
selected_entries = good_entries[:top_k]
if not selected_entries:
continue
kd_list = lmap(lambda x: x['kdp'], selected_entries)
qk_units.append((query, kd_list))
select_rate = len(selected_entries) / len(entries)
select_rate_list.append(select_rate)
print("{} of {} qk units selected".format(len(qk_units), len(grouped)))
print("average select rate", average(select_rate_list))
return qk_units
def main2():
counter = load_from_pickle(sys.argv[1])
couter_per_length = defaultdict(Counter)
l = []
for key, cnt in counter.items():
max_idx, num_seg = key
e = num_seg, max_idx, cnt
l.append(e)
grouped = group_by(l, get_first)
rows = []
for num_seg in range(1, 20):
entries = grouped[num_seg]
cnt_sum = sum([cnt for _, max_idx, cnt in entries])
local_counter = Counter()
for _, max_idx, cnt in entries:
local_counter[max_idx] = cnt
row = [num_seg, cnt_sum]
for seg_loc in range(num_seg):
row.append(local_counter[seg_loc])
rows.append(row)
print_table(rows)
def main():
score_d: Dict[Tuple[str, str, int],
float] = load_from_pickle("robust_score_d")
score_d2: Dict[Tuple[str, str, int],
float] = load_from_pickle("robust_score_d2")
qrel: Dict[str, Dict[str, int]] = load_robust_qrel()
query_grouped = group_by(score_d.keys(), get_first)
counter = Counter()
for query_id in query_grouped:
keys: List[Tuple[str, str, int]] = query_grouped[query_id]
doc_id_grouped = group_by(keys, get_second)
qrel_part = qrel[query_id] if query_id in qrel else {}
for doc_id in doc_id_grouped:
label: int = qrel_part[doc_id] if doc_id in qrel_part else 0
cur_keys: List[Tuple[str, str, int]] = doc_id_grouped[doc_id]
if len(cur_keys) == 1:
continue
summary = []
summary2 = []
for key in cur_keys:
query_id2, doc_id2, passage_idx = key
assert query_id2 == query_id
assert doc_id2 == doc_id
score = score_d[key]
score2 = score_d2[key]
summary.append((passage_idx, score))
summary2.append((passage_idx, score2))
summary.sort(key=get_first)
summary2.sort(key=get_first)
max_idx = int(argmax(right(summary)))
max_idx2 = int(argmax(right(summary2)))
if label:
if max_idx == max_idx2:
counter[1] += 1
else:
counter[0] += 1
print(counter)
accuracy = counter[1] / (counter[0] + counter[1])
print("accuracy {}".format(accuracy))
def dvp_to_correctness(dvp_list: List[DocValueParts],
run_config: Dict) -> Dict[Tuple[str, Tuple[str, int]], bool]:
def is_good(values: List[float]) -> bool:
# value = abs(error_new) - abs(error_baseline)
return sum(values) > 0
grouped = group_by(dvp_list, get_dvp_qid)
is_good_dict: Dict[Tuple[str, Tuple[str, int]], bool] = {}
for qid, entries in grouped.items():
g2 = group_by(entries, get_doc_id_idx)
for doc_sig, entries2 in g2.items():
values = lmap(get_dvp_value, entries2)
is_good_dict[qid, doc_sig] = is_good(values)
return is_good_dict
def num_files_to_touch():
doc_id_list = readlines_strip(
at_output_dir("clueweb", "not_found.sort.txt"))
grouped = group_by(doc_id_list, get_doc_group)
dir_helper = get_sydney_clueweb09_corpus_helper()
for group_id, doc_ids in grouped.items():
num_files = dir_helper.iter_gz_files_for_group(group_id)
print(len(doc_ids), len(num_files))
def main():
save_name = sys.argv[1]
score_d: Dict[Tuple[str, str, int], float] = load_from_pickle(save_name)
qrel: Dict[str, Dict[str, int]] = load_robust_qrel()
query_grouped = group_by(score_d.keys(), get_first)
for query_id in query_grouped:
keys: List[Tuple[str, str, int]] = query_grouped[query_id]
doc_id_grouped = group_by(keys, get_second)
qrel_part = qrel[query_id] if query_id in qrel else {}
pos_rows = []
neg_rows = []
for doc_id in doc_id_grouped:
label: int = qrel_part[doc_id] if doc_id in qrel_part else 0
cur_keys: List[Tuple[str, str, int]] = doc_id_grouped[doc_id]
summary = []
for key in cur_keys:
query_id2, doc_id2, passage_idx = key
assert query_id2 == query_id
assert doc_id2 == doc_id
score = score_d[key]
summary.append((passage_idx, score))
summary.sort(key=get_first)
max_idx = int(argmax(right(summary)))
score_str = list(["{0:.5f}".format(s) for s in right(summary)])
max_passage_idx = summary[max_idx][0]
row = [str(max_passage_idx)] + score_str
if label:
pos_rows.append(row)
else:
neg_rows.append(row)
print(query_id)
print("Positive")
print_table(pos_rows)
print("Negative")
print_table(neg_rows[:30])
def group_by_docs():
dvp: List[DocValueParts2] = load()
candidate_d_raw: List[Tuple[int, List[int]]] = get_eval_candidate_as_pids(
"train")
candidate_d = {str(k): lmap(str, v) for k, v in candidate_d_raw}
# Group by doc id
dvp_qid_grouped: Dict[str, List[DocValueParts2]] = group_by(dvp, get_qid)
def simple(doc_id):
return doc_id.split("-")[-1]
c_all = Counter()
rows = []
for qid, entries in dvp_qid_grouped.items():
# Q : How many kdp are useful?
# Q : Does relevant matter?
candidate_id_grouped = group_by(entries, get_doc_id)
rows.append(["qid", qid])
for doc_id_idx, entries2 in candidate_id_grouped.items():
#c = Counter([good_or_bad(e.score-e.init_score, e.label) for e in entries2])
c = Counter([
get_decision_change(e.label, e.init_score, e.score)
for e in entries2
])
rows.append([doc_id_idx])
#row = [doc_id_idx, c["good"], c["bad"], c["no change"]]
row = [
doc_id_idx, c["decision_change_good"],
c["decision_change_bad"], c["no_change"]
]
rows.append(row)
for k, v in c.items():
c_all[k] += v
row = [
"summary", c_all["decision_change_good"], c_all["decision_change_bad"],
c_all["no_change"]
]
rows = [row] + rows
print_table(rows)
def get_token_score_from_sspairs(
ss_pairs: List[SegmentScorePair]) -> Dict[int, List[SegmentScorePair]]:
grouped: Dict[Tuple[int, int], List[SegmentScorePair]] = group_by(
ss_pairs, SegmentScorePair.grouping_key)
token_to_pair_d = defaultdict(list)
for key, ss_pairs in grouped.items():
for ss_pair in ss_pairs:
for idx in range(ss_pair.abl_idx_st, ss_pair.abl_idx_ed):
token_to_pair_d[idx].append(ss_pair)
return token_to_pair_d
def group_by_cpid(info_dir, prediction_file) -> Dict[CPIDPair, List[Dict]]:
info = load_combine_info_jsons(info_dir)
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
scores: List[Dict] = collect_data_w_cpid(prediction_file, info,
logit_to_score_softmax)
grouped: Dict[CPIDPair, List[Dict]] = group_by(scores, lambda x: x['cpid'])
return grouped
def scrore_d_to_trec_style_predictions(score_d: Dict[CPIDPair, float], run_name="runname") -> List[TrecRankedListEntry]:
cid_grouped: Dict[int, List[Tuple[CPIDPair, float]]] = group_by(score_d.items(), lambda x: x[0][0])
for cid, entries in cid_grouped.items():
l: List[Tuple[CPIDPair, float]] = list(entries)
l.sort(key=get_second, reverse=True)
query_id = str(cid)
for rank, (cpid, score) in enumerate(l):
_, pid = cpid
doc_id = str(pid)
yield TrecRankedListEntry(query_id, doc_id, rank, score, run_name)
def get_best_seg_info_2d(self, job_id) -> Dict[str, Dict[str, int]]:
qdid_to_max_seg_idx = self.get_best_seg_info(job_id)
qdis: Dict[str,
List[Tuple[str,
str]]] = group_by(qdid_to_max_seg_idx.keys(),
get_first)
output = defaultdict(dict)
for qid, entries in qdis:
for qid_, doc_id in entries:
output[qid][doc_id] = qdid_to_max_seg_idx[qid, doc_id]
return output
def show_num_mention():
train, val = load_feature_and_split()
p_dict = get_perspective_dict()
claims = get_claims_from_ids(lmap(lambda x: x['cid'], train))
claim_d = claims_to_dict(claims)
grouped = group_by(train, lambda x: x['cid'])
for cid in grouped:
print("Claim:", claim_d[cid])
for dp in grouped[cid]:
p_text = p_dict[dp['pid']]
print(dp['label'], get_num_mention(dp), p_text)
def combine_ranked_list(ranked_list_list):
data = []
for d in ranked_list_list:
for query, ranked_list in d.items():
data.append((query, ranked_list))
new_d = {}
key_fn = lambda x: x[0]
for key, sub_data in group_by(data, key_fn).items():
ranked_list = right(sub_data)
new_d[key] = merge_ranked_list_list(ranked_list)
return new_d
def doc_score_predictions():
passage_score_path = "output/cppnc/qknc_val"
info = load_combine_info_jsons("output/cppnc/qknc_val.info", qk_convert_map)
data = join_prediction_with_info(passage_score_path, info)
grouped: Dict[str, List[Dict]] = group_by(data, lambda x: x['query'].query_id)
def get_score_from_logit(logits):
return scipy.special.softmax(logits)[1]
for cid, passages in grouped.items():
scores: List[float] = lmap(lambda d: get_score_from_logit(d['logits']), passages)
yield cid, scores
def main(config):
info_dir = config['info_path']
prediction_file = config['pred_path']
f_handler = get_format_handler("qck")
info = load_combine_info_jsons(info_dir, f_handler.get_mapping(),
f_handler.drop_kdp())
data: List[Dict] = join_prediction_with_info(prediction_file, info,
["data_id", "logits"])
out_entries: List[QCKOutEntry] = lmap(QCKOutEntry.from_dict, data)
qrel: Dict[str, Dict[str,
int]] = load_qrels_structured(config['qrel_path'])
def get_label(query_id, candi_id):
if candi_id in qrel[query_id]:
return qrel[query_id][candi_id]
else:
return 0
def logit_to_score_softmax(logit):
return scipy.special.softmax(logit)[1]
grouped: Dict[str,
List[QCKOutEntry]] = group_by(out_entries,
lambda x: x.query.query_id)
for query_id, items in grouped.items():
raw_kdp_list = [(x.kdp.doc_id, x.kdp.passage_idx) for x in items]
kdp_list = unique_list(raw_kdp_list)
raw_candi_id_list = [x.candidate.id for x in items]
candi_id_list = unique_list(raw_candi_id_list)
logit_d = {(x.candidate.id, (x.kdp.doc_id, x.kdp.passage_idx)):
x.logits
for x in items}
labels = [get_label(query_id, candi_id) for candi_id in candi_id_list]
head_row0 = [" "] + labels
head_row1 = [" "] + candi_id_list
rows = [head_row0, head_row1]
for kdp_sig in kdp_list:
row = [kdp_sig]
for candi_id in candi_id_list:
try:
score = logit_to_score_softmax(logit_d[candi_id, kdp_sig])
score_str = "{0:.2f}".format(score)
except KeyError:
score_str = "-"
row.append(score_str)
rows.append(row)
print(query_id)
print_table(rows)
def main(config):
info_path = sys.argv[1]
pred_path = sys.argv[2]
info = load_combine_info_jsons(info_path, True)
predictions = join_prediction_with_info(pred_path, info, silent=True)
out_entries: List[OutEntry] = lmap(OutEntry.from_dict, predictions)
g = group_by(out_entries, lambda x: x.doc_id)
for doc_id in g:
entries: List[OutEntry] = g[doc_id]
scores = list([logit_to_score_softmax(e.logits) for e in entries])
print(doc_id, max(scores))
