def main():
config = load_run_config()
job_prefix = config['job_prefix']
qk_candidate_name = config['qk_candidate_name']
score_type = config['score_type']
split = config['split']
lms: List[ClaimLM] = build_gold_lms_for_sub_split(split)
lm_pair_list = list([(str(lm.cid), lm.LM) for lm in lms])
def get_scorer():
if score_type == "log_odd":
assert False
elif score_type == "raw_prob":
return RawProbabilityScorer(lm_pair_list)
else:
assert False
scorer: ScorerInterface = get_scorer()
def get_score(q: QCKQuery, kdp: KDP) -> ScoreVector:
scores = []
for token in kdp.tokens:
score = scorer.score_token(q.query_id, token)
scores.append(score)
return scores
start_generate_jobs_for_sub_split(TokenScoringGen(get_score),
qk_candidate_name, job_prefix, split)
def make_test():
split = "test"
qk_candidate_name = "qk_candidate_msmarco_filtered_{}".format(split)
generator = QCKInstanceGenerator(get_eval_candidates_as_qck(split),
is_correct_factory())
start_generate_jobs_for_sub_split(generator, qk_candidate_name, "qcknc3",
split)
def main():
qk_candidate_name = "qk_candidate_msmarco_filtered_dev"
generator = QCKInstanceGenerator(get_eval_candidates_as_qck("dev"),
is_correct_factory())
start_generate_jobs_for_sub_split(generator, qk_candidate_name, "qcknc3",
"dev")
# Selected from doc_scorer_summarizer.py
qk_candidate_name = "qk_candidate_msmarco_filtered_train"
generator = QCKInstanceGenerator(get_eval_candidates_as_qck("train"),
is_correct_factory())
start_generate_jobs_for_train(generator, qk_candidate_name, "qcknc3")
generator = QCKInstanceGenerator(get_eval_candidates_as_qck("train"),
is_correct_factory())
start_generate_jobs_for_val(generator, qk_candidate_name, "qcknc3")
def main(config):
job_name = "qck_multi"
is_correct_fn = is_correct_factory()
qk_candidate_name = "qk_candidate_msmarco_filtered_dev"
generator = MultiDocInstanceGenerator(get_eval_candidates_as_qck("dev"),
is_correct_fn, config)
start_generate_jobs_for_sub_split(generator, qk_candidate_name, job_name,
"dev")
qk_candidate_name = "qk_candidate_msmarco_filtered_train"
generator = MultiDocInstanceGenerator(get_eval_candidates_as_qck("train"),
is_correct_fn, config)
start_generate_jobs_for_sub_split(generator, qk_candidate_name, job_name,
"train")
generator = MultiDocInstanceGenerator(get_eval_candidates_as_qck("train"),
is_correct_fn, config)
start_generate_jobs_for_sub_split(generator, qk_candidate_name, job_name,
"val")