def report_run_per_query(qrels,
run_file_name,
remove_docs_with_zero_score=False):
run = TrecRun(run_file_name)
system = run.run_data['system'][0]
if remove_docs_with_zero_score:
run.run_data = run.run_data[run.run_data['score'] > 0]
trec_eval = TrecEval(run, qrels)
bpref = trec_eval.getBpref(per_query=True)
ndcg_10 = trec_eval.getNDCG(depth=10, per_query='query')
ndcg = trec_eval.getNDCG(per_query='query')
ret = bpref.join(ndcg_10, on='query')
ret = ret.join(ndcg, on='query')
for query, r in ret.iterrows():
yield json.dumps({
'corpus': extract_corpus(run_file_name),
'topic': query,
'tag': system,
"bpref": r['Bpref@1000'],
"pseudoNDCG@10": r['NDCG@10'],
"pseudoNDCG": r['NDCG@1000']
})
def report_run(qrels, corpus, topics, run_file_name):
run = TrecRun(run_file_name)
trec_eval = TrecEval(run, qrels)
ret = {
'corpus': corpus,
'topics': topics,
'tag': run.run_data['system'][0],
"bpref": trec_eval.getBpref(),
"pseudoNDCG@10": trec_eval.getNDCG(depth=10, removeUnjudged=True),
"pseudoNDCG": trec_eval.getNDCG(removeUnjudged=True),
}
return json.dumps(ret)
def report_run(qrels, run_file_name, remove_docs_with_zero_score=False):
run = TrecRun(run_file_name)
system = run.run_data['system'][0]
if remove_docs_with_zero_score:
run.run_data = run.run_data[run.run_data['score'] > 0]
trec_eval = TrecEval(run, qrels)
ret = {
'corpus': extract_corpus(run_file_name),
'topics': extract_topics(run_file_name),
'tag': system,
"bpref": trec_eval.getBpref(),
"pseudoNDCG@10": trec_eval.getNDCG(depth=10),
"pseudoNDCG": trec_eval.getNDCG()
}
return json.dumps(ret)