def compute_map(valid_codes, pred, gs_out_path=None):
"""
Custom function to compute MAP evaluation metric.
Code adapted from https://github.com/TeMU-BSC/CodiEsp-Evaluation-Script/blob/master/codiespD_P_evaluation.py
"""
# Input args default values
if gs_out_path is None: gs_out_path = './intermediate_gs_file.txt'
pred_out_path = './intermediate_predictions_file.txt'
###### 2. Format predictions as TrecRun format: ######
format_predictions(pred, pred_out_path, valid_codes)
###### 3. Calculate MAP ######
# Load GS from qrel file
qrels = TrecQrel(gs_out_path)
# Load pred from run file
run = TrecRun(pred_out_path)
# Calculate MAP
te = TrecEval(run, qrels)
MAP = te.get_map(trec_eval=False) # With this option False, rank order is taken from the given document order
###### 4. Return results ######
return MAP
def eval(qrel_file_path, run_file_path):
"""[summary]
Arguments:
qrel_file_path {[string]} -- [path of the qrel file usually located at the source language folder]
run_file_path {[string]} -- [path of the run file usually located at the results folder of a language]
Returns:
[type] -- [precision@10, precision@20, precision@30, mAP rounded up to four digits]
"""
r1 = TrecRun(run_file_path)
qrels = TrecQrel(qrel_file_path)
te = TrecEval(r1, qrels)
p5 = te.get_precision(depth=5)
p10 = te.get_precision(depth=10)
p20 = te.get_precision(depth=20)
map = te.get_map()
rprec = te.get_rprec()
run_object = r1.evaluate_run(qrels, per_query=True)
return round(p5, 4), round(p10, 4), round(p20,
4), round(map,
4), round(rprec, 4)
def main(gs_path, pred_path, codes_path):
'''
Load GS, predictions and valid codes; format GS and predictions according
to TREC specifications; compute MAP and print it.
Parameters
----------
gs_path : str
Path to Gold Standard TSV with 2 columns: filename, code
It has no headers row.
pred_path : str
Path to Gold Standard TSV with 2 columns: filename, code
It has no headers row.
codes_path : str
Path to TSV file with valid codes.
It has no headers row.
Returns
-------
None.
'''
###### 0. Load valid codes lists: ######
valid_codes = set(
pd.read_csv(codes_path, sep='\t', header=None,
usecols=[0])[0].tolist())
valid_codes = set([x.lower() for x in valid_codes])
###### 1. Format GS as TrecQrel format: ######
qid_gs = format_gs(gs_path, './intermediate_gs_file.txt')
###### 2. Format predictions as TrecRun format: ######
format_predictions(pred_path, './intermediate_predictions_file.txt',
valid_codes, qid_gs)
###### 3. Calculate MAP ######
# Load GS from qrel file
qrels = TrecQrel('./intermediate_gs_file.txt')
# Load pred from run file
run = TrecRun('./intermediate_predictions_file.txt')
# Calculate MAP
te = TrecEval(run, qrels)
MAP = te.get_map(
trec_eval=False
) # With this option False, rank order is taken from the given document order
###### 4. Show results ######
print('\nMAP estimate: {}\n'.format(round(MAP, 3)))
#print('\n{}'.format(round(MAP, 3)))
print('{}|{}'.format(pred_path, round(MAP, 3)))
def trec_eval(runs_file_path: Path or str, qrels_file_path: Path or str):
metrics = dict()
r1 = TrecRun(str(runs_file_path.absolute()))
qrels = TrecQrel(str(qrels_file_path.absolute()))
results = TrecEval(r1, qrels)
metrics["P@5"] = results.get_precision(5)
metrics["P@10"] = results.get_precision(10)
metrics["P@15"] = results.get_precision(15)
metrics["bpref"] = results.get_bpref()
metrics["map"] = results.get_map()
metrics = {k: round(v, 4) for k, v in metrics.items()}
return metrics
def map(self):
qrels_file = TrecQrel("./Data/qrel.txt")
path_to_runs = TrecRun("./Data/run.txt")
te = TrecEval(path_to_runs, qrels_file)
dic = {"map": te.get_map(), "ndcg": te.get_ndcg()}
return dic
gs_path, pred_path, codes_path = parse_arguments()
###### 0. Load valid codes lists: ######
valid_codes = set(
pd.read_csv(codes_path, sep='\t', header=None,
usecols=[0])[0].tolist())
valid_codes = set([x.lower() for x in valid_codes])
###### 1. Format GS as TrecQrel format: ######
format_gs(gs_path, './intermediate_gs_file.txt')
###### 2. Format predictions as TrecRun format: ######
format_predictions(pred_path, './intermediate_predictions_file.txt',
valid_codes)
###### 3. Calculate MAP ######
# Load GS from qrel file
qrels = TrecQrel('./intermediate_gs_file.txt')
# Load pred from run file
run = TrecRun('./intermediate_predictions_file.txt')
# Calculate MAP
te = TrecEval(run, qrels)
MAP = te.get_map(
trec_eval=False
) # With this option False, rank order is taken from the given document order
###### 4. Show results ######
print('\nMAP estimate: {}\n'.format(round(MAP, 3)))
class TestTrecEval(unittest.TestCase):
def setUp(self):
run1 = TrecRun("./files/r4.run")
qrels1 = TrecQrel("./files/qrel1.txt")
run2 = TrecRun("./files/input.uic0301")
qrels2 = TrecQrel("./files/robust03_cs_qrels.txt")
# Contains the first 30 documents for the first 10 topics in input.uic0301
run3 = TrecRun("./files/input.uic0301_top30")
self.common_topics = [303, 307, 310, 314, 320, 322, 325, 330, 336, 341]
self.teval1 = TrecEval(run1, qrels1)
self.teval2 = TrecEval(run2, qrels2)
self.teval3 = TrecEval(run3, qrels2)
def tearDown(self):
pass
def test_get_reciprocal_rank(self):
value = self.teval1.get_reciprocal_rank(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.5000, places=4)
value = self.teval2.get_reciprocal_rank(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.6466, places=4)
values1 = self.teval2.get_reciprocal_rank(
depth=30, per_query=True,
trec_eval=True).loc[self.common_topics].values
values2 = self.teval3.get_reciprocal_rank(
depth=1000, per_query=True,
trec_eval=True).loc[self.common_topics].values
for v1, v2 in zip(values1, values2):
self.assertAlmostEqual(v1, v2, places=4)
results = self.teval2.get_reciprocal_rank(depth=1000,
trec_eval=True,
per_query=True)
correct_results = [0.0017, 0.1429, 0.3333]
values = map(float, results.loc[[378, 650, 624]].values)
for v, c in zip(values, correct_results):
self.assertAlmostEqual(v, c, places=4)
def test_get_map(self):
value = self.teval1.get_map(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.2685, places=4)
value = self.teval2.get_map(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.2396, places=4)
values1 = self.teval2.get_map(
depth=30, per_query=True,
trec_eval=True).loc[self.common_topics].values
values2 = self.teval3.get_map(
depth=1000, per_query=True,
trec_eval=True).loc[self.common_topics].values
for v1, v2 in zip(values1, values2):
self.assertAlmostEqual(v1, v2, places=4)
results = self.teval2.get_map(depth=1000,
trec_eval=True,
per_query=True)
correct_results = [0.4926, 0.2808, 0.2335]
values = map(float, results.loc[[622, 609, 320]].values)
for v, c in zip(values, correct_results):
self.assertAlmostEqual(v, c, places=4)
def test_get_precision(self):
value = self.teval1.get_precision(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.0010, places=4)
value = self.teval2.get_precision(depth=1000, trec_eval=True)
self.assertAlmostEqual(value, 0.0371, places=4)
values1 = self.teval2.get_precision(
depth=30, per_query=True,
trec_eval=True).loc[self.common_topics].values
values2 = self.teval3.get_precision(
depth=30, per_query=True,
trec_eval=True).loc[self.common_topics].values
for v1, v2 in zip(values1, values2):
self.assertAlmostEqual(v1, v2, places=4)
values1 = self.teval2.get_precision(
depth=500, per_query=True,
trec_eval=True).loc[self.common_topics].values
values2 = self.teval3.get_precision(
depth=500, per_query=True,
trec_eval=True).loc[self.common_topics].values
for v1, v2 in zip(values1, values2):
print(v1, v2)
self.assertNotAlmostEqual(float(v1), float(v2), places=4)
results = self.teval2.get_precision(depth=30,
trec_eval=True,
per_query=True)
correct_results = [0.1333, 0.0333, 0.5333]
values = map(float, results.loc[[607, 433, 375]].values)
for v, c in zip(values, correct_results):
self.assertAlmostEqual(v, c, places=4)
