def eval(topx): qrels = TrecQrel("./relevance-args-v2.qrels") # Generates a P@10 graph with all the runs in a directory path_to_runs = "./runs/" runs = procedures.list_of_runs_from_path(path_to_runs, "*") for run in runs: print(run.get_filename()) te = TrecEval(run, qrels) rbp, residuals = te.get_rbp() coverage = run.get_mean_coverage(qrels, topX=topx) print( "Average number of documents judged among top %.0f: %.2f, thats about: %.2f percent." % (topx, coverage, coverage / topx * 100)) # precision = te.get_precision(depth=topx) # print("precision (p"+str(topx)+"): ",precision) ndcg = te.get_ndcg(depth=topx, removeUnjudged=False) print("nDCG (n=" + str(topx) + " removeUnjudged=False): " + str(ndcg)) ndcg = te.get_ndcg(depth=topx, removeUnjudged=True) print("nDCG (n=" + str(topx) + " removeUnjudged=True): " + str(ndcg)) print("--------\n")
def eval_rank_df(e: trectools.TrecEval, per_query=True) -> pd.Series: return pd.Series({ "nDCG@5": e.get_ndcg(depth=5, per_query=per_query), "nDCG@10": e.get_ndcg(depth=10, per_query=per_query), "nDCG@20": e.get_ndcg(depth=20, per_query=per_query), "RR": e.get_reciprocal_rank(per_query=per_query) })
def evaluate(qrels, runs_file, topics, model): runs = TrecRun(runs_file) ev = TrecEval(runs, qrels) path_to_csv = os.path.join("eval", model, "results.csv") n_topics = len(topics) # Calculate various metrics for each query considering the runs/judgment files provided print("Calculating metrics...") res = ev.evaluate_all(per_query=True) # Write results of evaluation to csv file res.printresults(path_to_csv, "csv", perquery=True) # Calculate NDCG@100 for each query, since the previous metrics don't include it, # and append it to each line of the new csv file ndcgs = ev.get_ndcg(depth=100, per_query=True) values = [row['NDCG@100'] for i, row in ndcgs.iterrows() ] # Column name of Pandas dataframe storing the data with open(path_to_csv, 'r') as f: lines = [line[:-1] for line in f] # Remove '\n' from the end of each line lines[0] += ",ndcg@100\n" # Add new column to header for i in range( 1, n_topics + 1 ): # Lines 1 to n contain metric values for each of the n queries lines[i] += "," + str( values[i - 1] ) + "\n" # Line 1 (i) should store value 0 (i-1) - arrays start at 0 global_ndcg = ev.get_ndcg(depth=100, per_query=False) # Calculate global NDCG lines[n_topics + 1] += "," + str( global_ndcg) + "\n" # Append global NDCG to last line with open(path_to_csv, 'w') as f: f.writelines(lines) # Overwrite csv file with new content
def map(self): qrels_file = TrecQrel("./Data/qrel.txt") path_to_runs = TrecRun("./Data/run.txt") te = TrecEval(path_to_runs, qrels_file) dic = {"map": te.get_map(), "ndcg": te.get_ndcg()} return dic
def eval_rank_df(e: trectools.TrecEval) -> pd.Series: return pd.Series({ "nDCG@5": e.get_ndcg(depth=5, per_query=False), "nDCG@10": e.get_ndcg(depth=10, per_query=False), "nDCG@20": e.get_ndcg(depth=20, per_query=False), })