def pipeline(in_file,
eval_file=None,
model_path_dict=default_model_path_dict,
steps=default_steps):
"""
:param in_file: The raw input file.
:param eval_file: Whether to provide evaluation along the line.
:return:
"""
sentence_retri_1_scale_prob = 0.5
sentence_retri_2_scale_prob = 0.9
sent_retri_1_top_k = 5
sent_retri_2_top_k = 1
sent_prob_for_2doc = 0.1
sent_topk_for_2doc = 5
enhance_retri_1_scale_prob = -1
build_submission = True
doc_retrieval_method = 'word_freq'
haonan_docretri_object = HAONAN_DOCRETRI_OBJECT()
if not PIPELINE_DIR.exists():
PIPELINE_DIR.mkdir()
if steps['s1.tokenizing']['do']:
time_stamp = utils.get_current_time_str()
current_pipeline_dir = PIPELINE_DIR / f"{time_stamp}_r"
else:
current_pipeline_dir = steps['s1.tokenizing']['out_file'].parent
print("Current Result Root:", current_pipeline_dir)
if not current_pipeline_dir.exists():
current_pipeline_dir.mkdir()
eval_list = common.load_jsonl(eval_file) if eval_file is not None else None
in_file_stem = in_file.stem
tokenized_file = current_pipeline_dir / f"t_{in_file_stem}.jsonl"
# Save code into directory
script_name = os.path.basename(__file__)
with open(os.path.join(str(current_pipeline_dir), script_name),
'w') as out_f, open(__file__, 'r') as it:
out_f.write(it.read())
out_f.flush()
# Tokenizing.
print("Step 1. Tokenizing.")
if steps['s1.tokenizing']['do']:
tokenized_claim(in_file, tokenized_file) # Auto Saved
print("Tokenized file saved to:", tokenized_file)
else:
tokenized_file = steps['s1.tokenizing']['out_file']
print("Use preprocessed file:", tokenized_file)
# Tokenizing End.
# First Document retrieval.
print("Step 2. First Document Retrieval")
if steps['s2.1doc_retri']['do']:
doc_retrieval_result_list = first_doc_retrieval(
haonan_docretri_object,
tokenized_file,
method=doc_retrieval_method)
doc_retrieval_file_1 = current_pipeline_dir / f"doc_retr_1_{in_file_stem}.jsonl"
common.save_jsonl(doc_retrieval_result_list, doc_retrieval_file_1)
print("First Document Retrieval file saved to:", doc_retrieval_file_1)
else:
doc_retrieval_file_1 = steps['s2.1doc_retri']['out_file']
doc_retrieval_result_list = common.load_jsonl(doc_retrieval_file_1)
print("Use preprocessed file:", doc_retrieval_file_1)
if eval_list is not None:
print("Evaluating 1st Doc Retrieval")
eval_mode = {'check_doc_id_correct': True, 'standard': False}
print(
c_scorer.fever_score(doc_retrieval_result_list,
eval_list,
mode=eval_mode,
verbose=False))
# First Document retrieval End.
# First Sentence Selection.
print("Step 3. First Sentence Selection")
if steps['s3.1sen_select']['do']:
dev_sent_list_1_e0 = simple_nnmodel.pipeline_first_sent_selection(
tokenized_file, doc_retrieval_file_1, model_path_dict['sselector'])
dev_sent_file_1_e0 = current_pipeline_dir / f"dev_sent_score_1_{in_file_stem}.jsonl"
common.save_jsonl(dev_sent_list_1_e0, dev_sent_file_1_e0)
# Manual setting, delete it later
# dev_sent_file_1_e0 = None
# dev_sent_list_1_e0 = common.load_jsonl("/home/easonnie/projects/FunEver/results/pipeline_r/2018_07_24_11:07:41_r(new_model_v1_2_for_realtest)_scaled_0.05_selector_em/dev_sent_score_1_shared_task_test.jsonl")
# End
if steps['s3.1sen_select']['ensemble']:
print("Ensemble!")
dev_sent_list_1_e1 = simple_nnmodel.pipeline_first_sent_selection(
tokenized_file, doc_retrieval_file_1,
model_path_dict['sselector_1'])
dev_sent_file_1_e1 = current_pipeline_dir / f"dev_sent_score_1_{in_file_stem}_e1.jsonl"
common.save_jsonl(dev_sent_list_1_e1, dev_sent_file_1_e1)
# exit(0)
# dev_sent_list_1_e1 = common.load_jsonl(dev_sent_file_1_e1)
dev_sent_list_1_e2 = simple_nnmodel.pipeline_first_sent_selection(
tokenized_file, doc_retrieval_file_1,
model_path_dict['sselector_2'])
dev_sent_file_1_e2 = current_pipeline_dir / f"dev_sent_score_1_{in_file_stem}_e2.jsonl"
common.save_jsonl(dev_sent_list_1_e2, dev_sent_file_1_e2)
# exit(0)
# dev_sent_list_1_e2 = common.load_jsonl(dev_sent_file_1_e2)
dev_sent_list_1 = merge_sent_results(
[dev_sent_list_1_e0, dev_sent_list_1_e1, dev_sent_list_1_e2])
dev_sent_file_1 = current_pipeline_dir / f"dev_sent_score_1_{in_file_stem}_ensembled.jsonl"
common.save_jsonl(dev_sent_list_1, dev_sent_file_1)
# exit(0)
else:
dev_sent_list_1 = dev_sent_list_1_e0
dev_sent_file_1 = dev_sent_file_1_e0
# Merging two results
print("First Sentence Selection file saved to:", dev_sent_file_1)
else:
dev_sent_file_1 = steps['s3.1sen_select']['out_file']
dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
print("Use preprocessed file:", dev_sent_file_1)
# exit(0)
if eval_list is not None:
print("Evaluating 1st Sentence Selection")
# sent_select_results_list_1 = simi_sampler.threshold_sampler(tokenized_file, dev_sent_full_list,
# sentence_retri_scale_prob, top_n=5)
# additional_dev_sent_list = common.load_jsonl("/Users/Eason/RA/FunEver/results/sent_retri_nn/2018_07_20_15-17-59_r/dev_sent_2r.jsonl")
# dev_sent_full_list = dev_sent_full_list + additional_dev_sent_list
sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique(
tokenized_file,
dev_sent_list_1,
sentence_retri_1_scale_prob,
top_n=sent_retri_1_top_k)
# sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique_merge(sent_select_results_list_1,
# additional_dev_sent_list,
# sentence_retri_2_scale_prob,
# top_n=5, add_n=1)
eval_mode = {'check_sent_id_correct': True, 'standard': False}
# for a, b in zip(eval_list, sent_select_results_list_1):
# b['predicted_label'] = a['label']
print(
c_scorer.fever_score(sent_select_results_list_1,
eval_list,
mode=eval_mode,
verbose=False))
print("Step 4. Second Document Retrieval")
if steps['s4.2doc_retri']['do']:
dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
filtered_dev_instance_1_for_doc2 = simi_sampler.threshold_sampler_insure_unique(
tokenized_file,
dev_sent_list_1,
sent_prob_for_2doc,
top_n=sent_topk_for_2doc)
filtered_dev_instance_1_for_doc2_file = current_pipeline_dir / f"dev_sent_score_1_{in_file_stem}_scaled_for_doc2.jsonl"
common.save_jsonl(filtered_dev_instance_1_for_doc2,
filtered_dev_instance_1_for_doc2_file)
dev_sent_1_result = simi_sampler.threshold_sampler_insure_unique(
doc_retrieval_file_1, # Remember this name
dev_sent_list_1,
sentence_retri_1_scale_prob,
top_n=sent_topk_for_2doc)
dev_doc2_list = second_doc_retrieval(
haonan_docretri_object, filtered_dev_instance_1_for_doc2_file,
dev_sent_1_result)
dev_doc2_file = current_pipeline_dir / f"doc_retr_2_{in_file_stem}.jsonl"
common.save_jsonl(dev_doc2_list, dev_doc2_file)
print("Second Document Retrieval File saved to:", dev_doc2_file)
else:
dev_doc2_file = steps['s4.2doc_retri']['out_file']
# dev_doc2_list = common.load_jsonl(dev_doc2_file)
print("Use preprocessed file:", dev_doc2_file)
print("Step 5. Second Sentence Selection")
if steps['s5.2sen_select']['do']:
dev_sent_2_list = get_score_multihop(
tokenized_file,
dev_doc2_file,
model_path=model_path_dict['sselector'])
dev_sent_file_2 = current_pipeline_dir / f"dev_sent_score_2_{in_file_stem}.jsonl"
common.save_jsonl(dev_sent_2_list, dev_sent_file_2)
print("First Sentence Selection file saved to:", dev_sent_file_2)
else:
dev_sent_file_2 = steps['s5.2sen_select']['out_file']
if eval_list is not None:
print("Evaluating 1st Sentence Selection")
dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
dev_sent_list_2 = common.load_jsonl(dev_sent_file_2)
sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique(
tokenized_file,
dev_sent_list_1,
sentence_retri_1_scale_prob,
top_n=5)
sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique_merge(
sent_select_results_list_1,
dev_sent_list_2,
sentence_retri_2_scale_prob,
top_n=5,
add_n=sent_retri_2_top_k)
eval_mode = {'check_sent_id_correct': True, 'standard': False}
# for a, b in zip(eval_list, sent_select_results_list_1):
# b['predicted_label'] = a['label']
print(
c_scorer.fever_score(sent_select_results_list_1,
eval_list,
mode=eval_mode,
verbose=False))
# print("Step 6. NLI")
# if steps['s6.nli']['do']:
# dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
# dev_sent_list_2 = common.load_jsonl(dev_sent_file_2)
# sentence_retri_1_scale_prob = 0.05
# print("Threshold:", sentence_retri_1_scale_prob)
# sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique(tokenized_file, dev_sent_list_1,
# sentence_retri_1_scale_prob, top_n=5)
# # sent_select_results_list_2 = simi_sampler.threshold_sampler_insure_unique_merge(sent_select_results_list_1,
# # dev_sent_list_2,
# # sentence_retri_2_scale_prob,
# # top_n=5,
# # add_n=sent_retri_2_top_k)
# nli_results = nli.mesim_wn_simi_v1_2.pipeline_nli_run(tokenized_file,
# sent_select_results_list_1,
# [dev_sent_file_1, dev_sent_file_2],
# model_path_dict['nli'],
# with_logits=True,
# with_probs=True)
#
# nli_results_file = current_pipeline_dir / f"nli_r_{in_file_stem}.jsonl"
# common.save_jsonl(nli_results, nli_results_file)
# else:
# nli_results_file = steps['s6.nli']['out_file']
# nli_results = common.load_jsonl(nli_results_file)
# Ensemble code
# dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
# dev_sent_list_2 = common.load_jsonl(dev_sent_file_2)
# sentence_retri_1_scale_prob = 0.05
# print("NLI sentence threshold:", sentence_retri_1_scale_prob)
# sent_select_results_list_1 = simi_sampler.threshold_sampler_insure_unique(tokenized_file, dev_sent_list_1,
# sentence_retri_1_scale_prob, top_n=5)
#
# # sent_select_results_list_2 = simi_sampler.threshold_sampler_insure_unique_merge(sent_select_results_list_1,
# # dev_sent_list_2,
# # sentence_retri_2_scale_prob,
# # top_n=5,
# # add_n=sent_retri_2_top_k)
# # nli_results = nli.mesim_wn_simi_v1_2.pipeline_nli_run(tokenized_file,
# # sent_select_results_list_1,
# # [dev_sent_file_1, dev_sent_file_2],
# # model_path_dict['nli'], with_probs=True, with_logits=True)
#
# # nli_results = nli.mesim_wn_simi_v1_2.pipeline_nli_run_bigger(tokenized_file,
# # sent_select_results_list_1,
# # [dev_sent_file_1, dev_sent_file_2],
# # model_path_dict['nli_2'],
# # with_probs=True,
# # with_logits=True)
#
# nli_results = nli.mesim_wn_simi_v1_2.pipeline_nli_run_bigger(tokenized_file,
# sent_select_results_list_1,
# [dev_sent_file_1, dev_sent_file_2],
# model_path_dict['nli_4'],
# with_probs=True,
# with_logits=True)
#
# nli_results_file = current_pipeline_dir / f"nli_r_{in_file_stem}_withlb_e4.jsonl"
# common.save_jsonl(nli_results, nli_results_file)
# Ensemble code end
# exit(0)
nli_r_e0 = common.load_jsonl(current_pipeline_dir /
"nli_r_shared_task_test_withlb_e0.jsonl")
nli_r_e1 = common.load_jsonl(current_pipeline_dir /
"nli_r_shared_task_test_withlb_e1.jsonl")
nli_r_e2 = common.load_jsonl(current_pipeline_dir /
"nli_r_shared_task_test_withlb_e2.jsonl")
nli_r_e3 = common.load_jsonl(current_pipeline_dir /
"nli_r_shared_task_test_withlb_e3.jsonl")
nli_r_e4 = common.load_jsonl(current_pipeline_dir /
"nli_r_shared_task_test_withlb_e4.jsonl")
nli_results = merge_nli_results(
[nli_r_e0, nli_r_e1, nli_r_e2, nli_r_e3, nli_r_e4])
print("Post Processing enhancement")
delete_unused_evidence(nli_results)
print("Deleting Useless Evidence")
dev_sent_list_1 = common.load_jsonl(dev_sent_file_1)
dev_sent_list_2 = common.load_jsonl(dev_sent_file_2)
print("Appending 1 of second Evidence")
nli_results = simi_sampler.threshold_sampler_insure_unique_merge(
nli_results,
dev_sent_list_2,
sentence_retri_2_scale_prob,
top_n=5,
add_n=sent_retri_2_top_k)
delete_unused_evidence(nli_results)
# High tolerance enhancement!
print("Final High Tolerance Enhancement")
print("Appending all of first Evidence")
nli_results = simi_sampler.threshold_sampler_insure_unique_merge(
nli_results,
dev_sent_list_1,
enhance_retri_1_scale_prob,
top_n=100,
add_n=100)
delete_unused_evidence(nli_results)
if build_submission:
output_file = current_pipeline_dir / "predictions.jsonl"
build_submission_file(nli_results, output_file)
def pipeline_tokenize(in_file, out_file):
tokenized_claim(in_file, out_file)
def tokenization():
print("Start tokenizing dev and training set.")
tokenized_claim(config.FEVER_DEV_JSONL, config.T_FEVER_DEV_JSONL)
tokenized_claim(config.FEVER_TRAIN_JSONL, config.T_FEVER_TRAIN_JSONL)
print("Tokenization finished.")