def sample_answer_with_priority(self, d_list, top_k=5):
for i, item in enumerate(spcl(d_list)):
self.item_rb.rules(item)
item['predicted_docids'] = \
list(set([k for k, v \
in sorted(item['prioritized_docids'],
key=lambda x: (-x[1], x[0]))][:top_k]))
def sample_answer_with_priority(cls, d_list, top_k=5):
ws_first = websocket.create_connection(cls.first_round_path)
for i in spcl(range(len(d_list))):
# for i in spcl(range(10)):
ws_first.send(json.dumps(d_list[i]))
item = json.loads(ws_first.recv())
item['predicted_docids'] = \
list(set([k for k, v \
in sorted(item['prioritized_docids'],
key=lambda x: (-x[1], x[0]))][:top_k]))
d_list[i] = item
ws_first.close()
def tf_idf_rank(args, top_k=5):
dev_path = config.PRO_ROOT / \
'results_old/doc_retri/docretri.basic.nopageview/dev.jsonl'
cursor = get_cursor()
d_list = read_jsonl(dev_path)
d_list_test = d_list
for i, item in enumerate(spcl(d_list_test)):
all_sent = []
all_ids = [it[0] for it in item['prioritized_docids']]
try:
for doc_id in all_ids:
r_list, _ = get_all_sent_by_doc_id(cursor,
doc_id,
with_h_links=False)
all_sent.append(' '.join(r_list))
ranker = OnlineTfidfDocRanker(args, args.hash_size, args.ngram,
all_sent)
except Exception as e:
if i - 1 >= 0:
print(f'Early quit at {i-1} because of {e}')
save_path = config.RESULT_PATH / \
'doc_retri/docretri.tfidfrank/' \
f'dev_quit_dump_{uuid4()}.json'
DocRetrievalExperiment.dump_results(d_list_test[:i], save_path)
raise e
rank_ind, rank_score = \
ranker.closest_docs(' '.join(item['claim_tokens']), k=100)
id_score_dict = {docid: 0 for docid in all_ids}
id_score_dict.update({all_ids[ri]: rs \
for ri, rs in zip(rank_ind, rank_score)})
item['prioritized_docids'] = [(k, v) for k, v in id_score_dict.items()]
item['predicted_docids'] = \
list(set([k for k, v \
in sorted(item['prioritized_docids'],
key=lambda x: (-x[1], x[0]))][:top_k]))
save_path = config.RESULT_PATH / 'doc_retri/docretri.tfidfrank/dev.json'
DocRetrievalExperiment.dump_results(d_list_test, save_path)
def find_sent_link_with_priority(self, d_list, top_k=5, predict=False):
for item in spcl(d_list):
self.item_rb.second_only_rules(item)
pids = [it[0] for it in item['prioritized_docids']]
item['prioritized_docids_aside'] = \
[it for it in item['prioritized_docids_aside']\
if it[0] not in pids]
if predict:
porg = \
set([k for k, v \
in sorted(item['prioritized_docids'],
key=lambda x: (-x[1], x[0]))][:top_k])
paside = \
set([k for k, v \
in sorted(item['prioritized_docids_aside'],
key=lambda x: (-x[1], x[0]))][:top_k])
item['predicted_docids'] = list(porg | paside)
item['predicted_docids_origin'] = list(porg)
item['predicted_docids_aside'] = list(paside)
def main():
import os
from chaonan_src._config import old_result_path
from chaonan_src._utils.doc_utils import read_jsonl
from chaonan_src._utils.spcl import spcl
# pageview_path = os.path.join(config.RESULT_PATH,
# 'doc_retri/docretri.rawpageview/train.jsonl')
pageview_path = config.RESULT_PATH / \
'doc_retri/docretri.rawpageview/dev.jsonl'
# ori_path = os.path.join(old_result_path,
# 'doc_retri/docretri.pageview/dev.jsonl')
# d_list = read_jsonl(config.FEVER_DEV_JSONL)
# item_rb_exp = ItemRuleRawPageview()
# doc_exp = DocRetrievalExperiment(item_rb_exp)
# doc_exp.sample_answer_with_priority(d_list)
# doc_exp.dump_results(d_list, save_path)
# doc_exp.print_eval(d_list)
# d_list_ori = read_jsonl(ori_path)
d_list = read_jsonl(pageview_path)
# DocRetrievalExperiment.dump_results(d_list, pageview_path)
# item_rb = ItemRuleRawPageview()
top_k = 5
for item in spcl(d_list):
item['predicted_docids'] = \
list(set([k for k, v \
in sorted(item['docid_pageviews'],
key=lambda x: (-x[1], x[0]))][:top_k]))
# DocRetrievalExperiment.dump_results(d_list, save_path)
DocRetrievalExperiment.print_eval(d_list)
from IPython import embed
embed()
import os
os._exit(1)
def find_sent_link_with_priority(cls, d_list, top_k=5, predict=False):
ws_second = websocket.create_connection(cls.second_round_path)
for i in spcl(range(len(d_list))):
ws_second.send(json.dumps(d_list[i]))
item = json.loads(ws_second.recv())
pids = [it[0] for it in item['prioritized_docids']]
item['prioritized_docids_aside'] = \
[it for it in item['prioritized_docids_aside']\
if it[0] not in pids]
if predict:
porg = \
set([k for k, v \
in sorted(item['prioritized_docids'],
key=lambda x: (-x[1], x[0]))][:top_k])
paside = \
set([k for k, v \
in sorted(item['prioritized_docids_aside'],
key=lambda x: (-x[1], x[0]))][:top_k])
item['predicted_docids'] = list(porg | paside)
item['predicted_docids_origin'] = list(porg)
item['predicted_docids_aside'] = list(paside)
d_list[i] = item
ws_second.close()
def count_pageviews():
"""Count pageviews and save to a dictionary
"""
fname = save_path_root + get_file_name(7, 19, 12)
en_wiki_count = defaultdict(lambda: 0)
all_files = glob(save_path_root + '**/*.gz', recursive=True)
for fname in spcl(all_files, every=1):
with gzip.open(fname, mode='rb') as f:
for l in f:
if l.startswith(b'en'):
lsplits = l.split()
if len(lsplits) != 4:
continue
_, docid, count, _ = lsplits
docid = docid.decode('utf-8')
count = int(count)
en_wiki_count[docid] += count
print("All done! Cheers!")
pv_dump_path = str(config.RESULT_PATH) + "/chaonan99/pageviews.pkl"
pickle.dump(dict(en_wiki_count), open(pv_dump_path, 'wb'))
from IPython import embed
embed()