def test_convergence(self): categories = ml.recommend_categories(self.user) #print "Categories: ", categories picked_category = ml.sample_distribution(categories.items())[0] #print "picked category: ", picked_category picked_aggr = EventActionAggregate(user=self.user, category=picked_category) lst = [] ctree = CachedCategoryTree() parents = ctree.parents(picked_category) count = 0 while count < 100: count +=1 print "Round: %d\r"%count, sys.stdout.flush() # recommend a new set of categories recommendation_scores = ml.recommend_categories(self.user) cats = ml.sample_category_distribution(recommendation_scores.items(), settings.N) found_count = cats.count(picked_category) #print "Categories: ",cats #print "ID: ", picked_category.id cats = set(cats) cats.discard(picked_category.id) # # G(oto) picked category picked_aggr.g += found_count picked_aggr.save() # X all other categories for c in cats: if c in parents: continue try: eaa = EventActionAggregate.objects.get(user=self.user, category=c) except EventActionAggregate.DoesNotExist: eaa = EventActionAggregate(user=self.user, category=c) eaa.x += 1 eaa.save() lst.append(found_count*100.0/settings.N) plt.plot(lst,color="blue") plt.title("Rate of learning one category") plt.xlabel("Trials") plt.ylabel("% of all Recommendations") plt.savefig("learning/test_results/test.pdf") plt.cla() self.assertTrue(True)
def calculate_plot_metrics(self, number_of_recommendations=1): """ This method calculates and plots (currently precision and recall) metrics. """ precision = [] precision_set = [] recall = [] recall_set = [] for i in range(number_of_recommendations): print "In loop: ", i, "\r", sys.stdout.flush() #event_ids = [e.id for e in ml.recommend_events(self.user)] #event_category_id_dict = ml.get_categories(event_ids,'C') #event_category_ids = [event_category_id_dict[e] for e in event_ids] # just get category ids directly cats = ml.recommend_categories(self.user) event_categories = ml.sample_category_distribution( cats.items(), settings.N) event_categories = [[a.id] for a in event_categories] #print map(lambda l: map(self.get_category_string, l),event_category_ids) p, pres = self.calculate_precision_value(event_categories) precision.append(p) precision_set.append(pres) r, rres = self.calculate_recall_value(event_categories) recall.append(r) recall_set.append(rres) self.update_behavior(event_categories) #print "Events: ", events #print "precision: ", precision #print "recall: ", recall #print [[self.get_category_string(c) for c in clst] for clst in event_category_ids][0] print "precision: ", precision #print "Precision: ", precision_set print "recall: ", recall print "recall: ", [ map(self.get_category_string, a) for a in map(list, recall_set) ] #plt.figure(1) #plt.plot(precision,color=color,label=label) #plt.figure(2) #plt.plot(recall,color=color, label=label) return (precision, recall)
def calculate_plot_metrics(self, number_of_recommendations=1): """ This method calculates and plots (currently precision and recall) metrics. """ precision = [] precision_set = [] recall = [] recall_set = [] for i in range(number_of_recommendations): print "In loop: ", i, "\r", sys.stdout.flush() #event_ids = [e.id for e in ml.recommend_events(self.user)] #event_category_id_dict = ml.get_categories(event_ids,'C') #event_category_ids = [event_category_id_dict[e] for e in event_ids] # just get category ids directly cats = ml.recommend_categories(self.user) event_categories = ml.sample_category_distribution(cats.items(), settings.N) event_categories= [[a.id] for a in event_categories] #print map(lambda l: map(self.get_category_string, l),event_category_ids) p,pres =self.calculate_precision_value(event_categories) precision.append(p) precision_set.append(pres) r,rres = self.calculate_recall_value(event_categories) recall.append(r) recall_set.append(rres) self.update_behavior(event_categories) #print "Events: ", events #print "precision: ", precision #print "recall: ", recall #print [[self.get_category_string(c) for c in clst] for clst in event_category_ids][0] print "precision: ", precision #print "Precision: ", precision_set print "recall: ", recall print "recall: ", [map(self.get_category_string,a) for a in map(list,recall_set)] #plt.figure(1) #plt.plot(precision,color=color,label=label) #plt.figure(2) #plt.plot(recall,color=color, label=label) return (precision, recall)
def test_multi_category_recall(self,user=None): if not user: user = self.user for c in Category.objects.all(): EventActionAggregate(user=user,category=c).save() # Each color corresponds to the number of categories (position in list + 1) selected during the iteration. colors = ["r","b","g","k","c","m","y"] # k is the number of categories selected. for k in range(1,8): categories = set(ml.recommend_categories(user)) picked_categories = set(random.sample(categories,k)) print "User picked categories: ", picked_categories # Generate mapping between categories and User's event action aggregate (GVIX store) picked_cat_aggregates = dict([(c,EventActionAggregate.objects.get(user=user, category=c)) for c in picked_categories]) trials = count() lst = [] while trials.next() < 50: print "Loop: ", trials mean_loop_count = count() recall = 0 iterations = 10 G = {} #temporary dictionary that store G counts for selected categories during iterations. while mean_loop_count.next() < iterations: cats = set(ml.recommend_categories(user)) correct_recommendations = cats.intersection(picked_categories) for c in correct_recommendations: try: G[c] += 1 except: G[c] = 1 #picked_cat_aggregates[x].save() cats.discard(picked_cat_aggregates[c]) #end of looping over correct_recommendations recall += len(correct_recommendations)*100.0/len(picked_categories) #end of looping over iterations to calculate means. ##### #print "Recall: ", recall #set and save the x values of discarded categories to the average number of times the category was G'd over iterations. for key,value in G.iteritems(): picked_cat_aggregates[key].g += min(round(value/iterations),2) picked_cat_aggregates[key].save() #end of looping over temporary dictionary. lst.append(recall*1.0/iterations) #Variant 1: All items in the last iterations that were not in the picked category have been X'd once. for c in cats: try: eaa = EventActionAggregate.objects.get(user=user, category=c) except EventActionAggregate.DoesNotExist: eaa = EventActionAggregate(user=user, category=c) eaa.x += 1 eaa.save() #end of adding X's #end print "Recall: ",lst plt.plot(lst,color=colors[k-1],label=k) for c in Category.objects.all(): #import pdb; pdb.set_trace() try: eaa = EventActionAggregate.objects.get(user=user, category=c) eaa.g = 0 eaa.v = 0 eaa.i = 0 eaa.x = 0 eaa.save() except: pass plt.title("Recall") plt.xlabel("Trials") plt.ylabel("% of User preferred categories") #plt.legend() plt.savefig("learning/test_results/recall.pdf") plt.cla() #import pdb; pdb.set_trace() self.assertTrue(True)