Example #1
0
 def rank_eval(self, map_not_ndcg=True, plot=False):
   c_ap, r_ap = [], []
   c_ndcg, r_ndcg = [], []
   for i in self.test_data:
     res, p_res = i["result"], i["predicted_result"]["shuffled_result"]
     print i["q"], " ", i["l"]
     labels = deepcopy(res["r"][:11])
     rank_as_per_prediction = deepcopy(p_res["r"])
     if map_not_ndcg:
       _, ap = Measures.avg_precision(labels)
       c_ap.append(ap)
       _, ap = Measures.avg_precision(rank_as_per_prediction)
       r_ap.append(ap)
     else:
       print "current: ", res["name"][:11]#labels
       print "after prediction: ", p_res["name"]#rank_as_per_prediction
       ideal = sorted(labels, reverse=True)
       print "ideal: ", ideal
       c_ndcg.append(Measures.ndcg(labels, ideal))
       r_ndcg.append(Measures.ndcg(rank_as_per_prediction, ideal))
   if map_not_ndcg:
     print "mean avg precision current: ", sum(c_ap)/len(c_ap)
     print "mean avg precision after ranking: ", sum(r_ap)/len(r_ap)
   else:
     print c_ndcg
     print r_ndcg
     print "mean ndcg current: ", sum(c_ndcg)/len(c_ndcg)
     print "mean ndcg after ranking: ", sum(r_ndcg)/len(r_ndcg)
     if plot:
       queries = [x["q"]+" "+x["l"] for x in self.test_data]
       plot_values([c_ndcg, r_ndcg], queries,
                   ["NDCG Fixed Rank", "NDCG Learned Rank"],
                   "NDCG Comparison before and after ranking model",
                   plot_means=True)
Example #2
0
 def get_map_and_ndcg(self, plot=True):
   results = []
   l = len(self.queries_data)
   for i in range(l):
     result = {"q": self.queries_data[i]["q"], "l": self.queries_data[i]["l"],
               "res": {}}
     for m in self.results:
       labels = []
       if m == "ole_lr":
         res = self.results[m][i]["predicted_result"]["shuffled_result"]["r"]
         labels.extend(res)
       else:
         res = self.results[m][i]["local_global_012"]
         labels.extend([float(x)+1 for x in res])
       res_m = result["res"][m] = {}
       (res_m["p"], res_m["ap"]) = Measures.avg_precision(labels)
       res_m["ndcg"] = Measures.ndcg(labels, sorted(labels, reverse=True))
     results.append(result)
   if plot:
     #Evaluation.plot_prec_at_10(results)
     #Evaluation.plot_compare_methods_map(results)
     #Evaluation.plot_compare_methods_ndcg(results)
     Evaluation.plot_compare_methods(results)
   #Evaluation.save_results_to_csv(results, "map_ndcg_results.csv")
   return results
Example #3
0
 def relevance_judgments(self):
   total_rel_judgments, total_rel = {}, {}
   l = len(self.queries_data)
   for i in range(l):
     for m in self.results:
       labels = []
       if m == "ole_lr":
         res = self.results[m][i]["predicted_result"]["shuffled_result"]["r"]
         labels.extend(res)
       else:
         res = self.results[m][i]["local_global_012"]
         labels.extend([float(x)+1 for x in res])
       (num_rel_judgments, num_rel) = Measures.relevance_judgments(labels)
       if m in total_rel_judgments:
         total_rel_judgments[m] += num_rel_judgments
         total_rel[m] += num_rel
       else:
         total_rel_judgments[m] = num_rel_judgments
         total_rel[m] = num_rel
   print "Total no. of relevance judgments: ", total_rel_judgments
   print "Total no. of relevant replies: ", total_rel