def evaluate(qrels, runs_file, topics, model):
runs = TrecRun(runs_file)
ev = TrecEval(runs, qrels)
path_to_csv = os.path.join("eval", model, "results.csv")
n_topics = len(topics)
# Calculate various metrics for each query considering the runs/judgment files provided
print("Calculating metrics...")
res = ev.evaluate_all(per_query=True)
# Write results of evaluation to csv file
res.printresults(path_to_csv, "csv", perquery=True)
# Calculate NDCG@100 for each query, since the previous metrics don't include it,
# and append it to each line of the new csv file
ndcgs = ev.get_ndcg(depth=100, per_query=True)
values = [row['NDCG@100'] for i, row in ndcgs.iterrows()
] # Column name of Pandas dataframe storing the data
with open(path_to_csv, 'r') as f:
lines = [line[:-1]
for line in f] # Remove '\n' from the end of each line
lines[0] += ",ndcg@100\n" # Add new column to header
for i in range(
1, n_topics + 1
): # Lines 1 to n contain metric values for each of the n queries
lines[i] += "," + str(
values[i - 1]
) + "\n" # Line 1 (i) should store value 0 (i-1) - arrays start at 0
global_ndcg = ev.get_ndcg(depth=100,
per_query=False) # Calculate global NDCG
lines[n_topics + 1] += "," + str(
global_ndcg) + "\n" # Append global NDCG to last line
with open(path_to_csv, 'w') as f:
f.writelines(lines) # Overwrite csv file with new content
def evaluate_run(self, trec_qrel_obj, per_query):
from trectools import TrecEval
evaluator = TrecEval(self, trec_qrel_obj)
result = evaluator.evaluate_all(per_query)
return result