def rank_eval(self, map_not_ndcg=True, plot=False):
c_ap, r_ap = [], []
c_ndcg, r_ndcg = [], []
for i in self.test_data:
res, p_res = i["result"], i["predicted_result"]["shuffled_result"]
print i["q"], " ", i["l"]
labels = deepcopy(res["r"][:11])
rank_as_per_prediction = deepcopy(p_res["r"])
if map_not_ndcg:
_, ap = Measures.avg_precision(labels)
c_ap.append(ap)
_, ap = Measures.avg_precision(rank_as_per_prediction)
r_ap.append(ap)
else:
print "current: ", res["name"][:11]#labels
print "after prediction: ", p_res["name"]#rank_as_per_prediction
ideal = sorted(labels, reverse=True)
print "ideal: ", ideal
c_ndcg.append(Measures.ndcg(labels, ideal))
r_ndcg.append(Measures.ndcg(rank_as_per_prediction, ideal))
if map_not_ndcg:
print "mean avg precision current: ", sum(c_ap)/len(c_ap)
print "mean avg precision after ranking: ", sum(r_ap)/len(r_ap)
else:
print c_ndcg
print r_ndcg
print "mean ndcg current: ", sum(c_ndcg)/len(c_ndcg)
print "mean ndcg after ranking: ", sum(r_ndcg)/len(r_ndcg)
if plot:
queries = [x["q"]+" "+x["l"] for x in self.test_data]
plot_values([c_ndcg, r_ndcg], queries,
["NDCG Fixed Rank", "NDCG Learned Rank"],
"NDCG Comparison before and after ranking model",
plot_means=True)
def get_map_and_ndcg(self, plot=True):
results = []
l = len(self.queries_data)
for i in range(l):
result = {"q": self.queries_data[i]["q"], "l": self.queries_data[i]["l"],
"res": {}}
for m in self.results:
labels = []
if m == "ole_lr":
res = self.results[m][i]["predicted_result"]["shuffled_result"]["r"]
labels.extend(res)
else:
res = self.results[m][i]["local_global_012"]
labels.extend([float(x)+1 for x in res])
res_m = result["res"][m] = {}
(res_m["p"], res_m["ap"]) = Measures.avg_precision(labels)
res_m["ndcg"] = Measures.ndcg(labels, sorted(labels, reverse=True))
results.append(result)
if plot:
#Evaluation.plot_prec_at_10(results)
#Evaluation.plot_compare_methods_map(results)
#Evaluation.plot_compare_methods_ndcg(results)
Evaluation.plot_compare_methods(results)
#Evaluation.save_results_to_csv(results, "map_ndcg_results.csv")
return results
def relevance_judgments(self):
total_rel_judgments, total_rel = {}, {}
l = len(self.queries_data)
for i in range(l):
for m in self.results:
labels = []
if m == "ole_lr":
res = self.results[m][i]["predicted_result"]["shuffled_result"]["r"]
labels.extend(res)
else:
res = self.results[m][i]["local_global_012"]
labels.extend([float(x)+1 for x in res])
(num_rel_judgments, num_rel) = Measures.relevance_judgments(labels)
if m in total_rel_judgments:
total_rel_judgments[m] += num_rel_judgments
total_rel[m] += num_rel
else:
total_rel_judgments[m] = num_rel_judgments
total_rel[m] = num_rel
print "Total no. of relevance judgments: ", total_rel_judgments
print "Total no. of relevant replies: ", total_rel