def Evaluate(self, evaluationData, doTopN, n=10, verbose=True): metrics = {} # Compute accuracy if (verbose): print("Evaluating accuracy...") self.algorithm.fit(evaluationData.GetTrainSet()) predictions = self.algorithm.test(evaluationData.GetTestSet()) metrics["RMSE"] = RecommenderMetrics.RMSE(predictions) metrics["MAE"] = RecommenderMetrics.MAE(predictions) if (doTopN): # Evaluate top-10 with Leave One Out testing if (verbose): print("Evaluating top-N with leave-one-out...") self.algorithm.fit(evaluationData.GetLOOCVTrainSet()) leftOutPredictions = self.algorithm.test( evaluationData.GetLOOCVTestSet()) # Build predictions for all ratings not in the training set allPredictions = self.algorithm.test( evaluationData.GetLOOCVAntiTestSet()) # Compute top 10 recs for each user topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) if (verbose): print("Computing hit-rate and rank metrics...") # See how often we recommended a movie the user actually rated metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions) # See how often we recommended a movie the user actually liked metrics["cHR"] = RecommenderMetrics.CumulativeHitRate( topNPredicted, leftOutPredictions) # Compute ARHR metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank( topNPredicted, leftOutPredictions) #Evaluate properties of recommendations on full training set if (verbose): print("Computing recommendations with full data set...") self.algorithm.fit(evaluationData.GetFullTrainSet()) allPredictions = self.algorithm.test( evaluationData.GetFullAntiTestSet()) topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) if (verbose): print("Analyzing coverage, diversity, and novelty...") # Print user coverage with a minimum predicted rating of 4.0: metrics["Coverage"] = RecommenderMetrics.UserCoverage( topNPredicted, evaluationData.GetFullTrainSet().n_users, ratingThreshold=4.0) # Measure diversity of recommendations: metrics["Diversity"] = RecommenderMetrics.Diversity( topNPredicted, evaluationData.GetSimilarities()) # Measure novelty (average popularity rank of recommendations): metrics["Novelty"] = RecommenderMetrics.Novelty( topNPredicted, evaluationData.GetPopularityRankings()) if (verbose): print("Analysis complete.") return metrics
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True): metrics = {} # Compute accuracy if (verbose): print("Evaluating accuracy...") self.algorithm.fit(evaluationData.GetTrainSet()) predictions = self.algorithm.test(evaluationData.GetTestSet()) metrics["RMSE"] = RecommenderMetrics.RMSE(predictions) metrics["MAE"] = RecommenderMetrics.MAE(predictions) if (doTopN): # Evaluate top-10 with Leave One Out testing if (verbose): print("leave-one-out法でtop-Nを計算...") self.algorithm.fit(evaluationData.GetLOOCVTrainSet()) leftOutPredictions = self.algorithm.test( evaluationData.GetLOOCVTestSet()) # 訓練データではない方で allPredictions = self.algorithm.test( evaluationData.GetLOOCVAntiTestSet()) # 各ユーザーに10個の表示を行う topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) if (verbose): print("Computing hit-rate and rank metrics...") # HitRate: どのくらいの頻度で、ユーザーが評価しうているかの指標 metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions) # cHitRate: ユーザーが実際にlikedしたかを確認する metrics["cHR"] = RecommenderMetrics.CumulativeHitRate( topNPredicted, leftOutPredictions) # ARHR metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank( topNPredicted, leftOutPredictions) #すべての訓練データセットでのレコメンデーションの特徴量を評価する if (verbose): print("すべてのデータセットでレコメンドの計算をしています...") self.algorithm.fit(evaluationData.GetFullTrainSet()) allPredictions = self.algorithm.test( evaluationData.GetFullAntiTestSet()) topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) if (verbose): print("Analyzing coverage, diversity, and novelty...") # 4.0を閾値として表示する metrics["Coverage"] = RecommenderMetrics.UserCoverage( topNPredicted, evaluationData.GetFullTrainSet().n_users, ratingThreshold=4.0) # レコメンドの多様性の測定 metrics["Diversity"] = RecommenderMetrics.Diversity( topNPredicted, evaluationData.GetSimilarities()) # レコメンドされた人気の高い評価の平均値を計算する metrics["Novelty"] = RecommenderMetrics.Novelty( topNPredicted, evaluationData.GetPopularityRankings()) if (verbose): print("Analysis complete.") return metrics
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True): metrics = {} # Compute accuracy if (verbose): print("Evaluating: ", self.name) self.algorithm.fit(evaluationData.GetTrainSet()) predictions = self.algorithm.test(evaluationData.GetTestSet()) metrics["RMSE"] = RecommenderMetrics.RMSE(predictions) metrics["MAE"] = RecommenderMetrics.MAE(predictions) if (doTopN): # Evaluate top-10 with Leave One Out testing self.algorithm.fit(evaluationData.GetLOOCVTrainSet()) leftOutPredictions = self.algorithm.test( evaluationData.GetLOOCVTestSet()) # Build predictions for all ratings not in the training set allPredictions = self.algorithm.test( evaluationData.GetLOOCVAntiTestSet()) # Compute top 10 recs for each user topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) # See how often we recommended a movie the user actually rated metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions) #Evaluate properties of recommendations on full training set self.algorithm.fit(evaluationData.GetFullTrainSet()) allPredictions = self.algorithm.test( evaluationData.GetFullAntiTestSet()) topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) # Print user coverage with a minimum predicted rating of 6.0: metrics["Coverage"] = RecommenderMetrics.UserCoverage( topNPredicted, evaluationData.GetFullTrainSet().n_users, ratingThreshold=7.0) # Measure novelty (average popularity rank of recommendations): metrics["Novelty"] = RecommenderMetrics.Novelty( topNPredicted, evaluationData.GetPopularityRankings()) return metrics
def get_hit_rates(trainSet, testSet, simsMatrix, num_nearest_neighbors=10): # Build up dict to lists of (int(movieID), predictedrating) pairs topN = defaultdict(list) k = num_nearest_neighbors for uiid in range(trainSet.n_users): # Get top N similar users to this one similarityRow = simsMatrix[uiid] similarUsers = [] for innerID, score in enumerate(similarityRow): if (innerID != uiid): similarUsers.append((innerID, score)) kNeighbors = heapq.nlargest(k, similarUsers, key=lambda t: t[1]) # Get the stuff they rated, and add up ratings for each item, weighted by user similarity candidates = defaultdict(float) for similarUser in kNeighbors: innerID = similarUser[0] userSimilarityScore = similarUser[1] theirRatings = trainSet.ur[innerID] for rating in theirRatings: candidates[rating[0]] += (rating[1] / 5.0) * userSimilarityScore # Build a dictionary of stuff the user has already seen watched = {} for itemID, rating in trainSet.ur[uiid]: watched[itemID] = 1 # Get top-rated items from similar users: pos = 0 for itemID, ratingSum in sorted(candidates.items(), key=itemgetter(1), reverse=True): if not itemID in watched: movieID = trainSet.to_raw_iid(itemID) topN[int(trainSet.to_raw_uid(uiid))].append( (int(movieID), 0.0)) pos += 1 if (pos > 40): break HR = RecommenderMetrics.HitRate(topN, testSet) CHR = RecommenderMetrics.CumulativeHitRate(topN, testSet, ratingCutoff=4.0) RHR = RecommenderMetrics.RatingHitRate(topN, testSet) ARHR = RecommenderMetrics.AverageReciprocalHitRank(topN, testSet) return HR, CHR, RHR, ARHR
# Predicts ratings for left-out ratings only print("Predict ratings for left-out set...") leftOutPredictions = algo.test(testSet) # Build predictions for all ratings not in the training set print("Predict all missing ratings...") bigTestSet = trainSet.build_anti_testset() allPredictions = algo.test(bigTestSet) # Compute top 10 recs for each user print("Compute top 10 recs per user...") topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n=10) # See how often we recommended a movie the user actually rated print("\nHit Rate: ", RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions)) # Break down hit rate by rating value print("\nrHR (Hit Rate by Rating value): ") RecommenderMetrics.RatingHitRate(topNPredicted, leftOutPredictions) # See how often we recommended a movie the user actually liked print("\ncHR (Cumulative Hit Rate, rating >= 4): ", RecommenderMetrics.CumulativeHitRate(topNPredicted, leftOutPredictions, 4.0)) # Compute ARHR print("\nARHR (Average Reciprocal Hit Rank): ", RecommenderMetrics.AverageReciprocalHitRank(topNPredicted, leftOutPredictions)) print("\nComputing complete recommendations, no hold outs...") algo.fit(fullTrainSet) bigTestSet = fullTrainSet.build_anti_testset() allPredictions = algo.test(bigTestSet)
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True, load=False): metrics = {} # Compute accuracy if (verbose): print("Evaluating accuracy...") pr = None alg = None if load: file_name = self.name + '_acc' pr, alg, _ = MyDump.Load(file_name, 1) if pr == None or alg == None: self.algorithm.fit(evaluationData.GetTrainSet()) # 75% predictions = self.algorithm.test(evaluationData.GetTestSet()) if load: MyDump.Save(file_name, predictions, self.algorithm, None, 1) else: self.algorithm = alg predictions = pr metrics["RMSE"] = RecommenderMetrics.RMSE(predictions) metrics["MAE"] = RecommenderMetrics.MAE(predictions) # if load: # MyDump.Save(file_name, predictions, self.algorithm, None, 1) if (doTopN): # if False: # Evaluate top-10 with Leave One Out testing if (verbose): print("Evaluating top-N with leave-one-out...") pr_top = None alg_top = None if load: file_name = self.name + '_top' + str(n) pr_top, alg_top, _ = MyDump.Load(file_name, 1) if pr_top == None or alg_top == None: self.algorithm.fit(evaluationData.GetLOOCVTrainSet()) leftOutPredictions = self.algorithm.test( evaluationData.GetLOOCVTestSet()) # Build predictions for all ratings not in the training set allPredictions = self.algorithm.test( evaluationData.GetLOOCVAntiTestSet()) # Compute top 10 recs for each user topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) pr_data = { 'leftOutPredictions': leftOutPredictions, 'topNPredicted': topNPredicted } if load: MyDump.Save(file_name, pr_data, self.algorithm, None, 1) else: self.algorithm = alg_top leftOutPredictions = pr_top['leftOutPredictions'] topNPredicted = pr_top['topNPredicted'] if (verbose): print("Computing hit-rate and rank metrics...") # See how often we recommended a movie the user actually rated metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions) # See how often we recommended a movie the user actually liked metrics["cHR"] = RecommenderMetrics.CumulativeHitRate( topNPredicted, leftOutPredictions) # Compute ARHR metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank( topNPredicted, leftOutPredictions) #Evaluate properties of recommendations on full training set if (verbose): print("Computing recommendations with full data set...") pr_top = None alg_top = None if load: file_name = self.name + '_fulldata' pr_top, alg_top, _ = MyDump.Load(file_name, 1) if pr_top == None or alg_top == None: self.algorithm.fit(evaluationData.GetFullTrainSet()) allPredictions = self.algorithm.test( evaluationData.GetFullAntiTestSet()) topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n) pr_data = { 'allPredictions': allPredictions, 'topNPredicted': topNPredicted } if load: MyDump.Save(file_name, pr_data, self.algorithm, None, 1) else: self.algorithm = alg_top allPredictions = pr_top['allPredictions'] topNPredicted = pr_top['topNPredicted'] if (verbose): print("Analyzing coverage, diversity, and novelty...") # Print user coverage with a minimum predicted rating of 4.0: metrics["Coverage"] = RecommenderMetrics.UserCoverage( topNPredicted, evaluationData.GetFullTrainSet().n_users, ratingThreshold=4.0) # Measure diversity of recommendations: metrics["Diversity"] = RecommenderMetrics.Diversity( topNPredicted, evaluationData.GetSimilarities()) # Measure novelty (average popularity rank of recommendations): metrics["Novelty"] = RecommenderMetrics.Novelty( topNPredicted, evaluationData.GetPopularityRankings()) if (verbose): print("Analysis complete.") # SaveAlg() return metrics
kNeighbors = heapq.nlargest(k, similarUsers, key=lambda t: t[1]) # Get the stuff they rated, and add up ratings for each item, weighted by user similarity candidates = defaultdict(float) for similarUser in kNeighbors: innerID = similarUser[0] userSimilarityScore = similarUser[1] theirRatings = trainSet.ur[innerID] for rating in theirRatings: candidates[rating[0]] += (rating[1] / 5.0) * userSimilarityScore # Build a dictionary of stuff the user has already seen watched = {} for itemID, rating in trainSet.ur[uiid]: watched[itemID] = 1 # Get top-rated items from similar users: pos = 0 for itemID, ratingSum in sorted(candidates.items(), key=itemgetter(1), reverse=True): if not itemID in watched: movieID = trainSet.to_raw_iid(itemID) topN[int(trainSet.to_raw_uid(uiid))].append( (int(movieID), 0.0) ) pos += 1 if (pos > 40): break # Measure print("HR", RecommenderMetrics.HitRate(topN, leftOutTestSet))
# Build predictions for all ratings not in the training set print(BeginGREEN + "Predict all missing ratings..." + EndGREEN) bigTestSet = trainSet.build_anti_testset() allPredictions = algo.test(bigTestSet) # Compute top 10 recs for each user print(BeginGREEN + "Compute top 10 recs per user..." + EndGREEN) topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n=10) print("\n") # See how often we recommended a movie the user actually rated print( BeginBgBLUE + "# Hit Rate; how often we are able to recommend a left-out rating. Higher is better. #" + EndBgBLUE) print(BeginBLUE + "Hit Rate: " + EndBLUE, BeginYELLO + "", RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions), "" + EndYELLO) print("\n") # Break down hit rate by rating value print(BeginBgBLUE + "# rating Hit Rate #" + EndBgBLUE) print(BeginBLUE + "rHR (Hit Rate by Rating value): " + EndBLUE) RecommenderMetrics.RatingHitRate(topNPredicted, leftOutPredictions) print("\n") # See how often we recommended a movie the user actually liked print( BeginBgBLUE + "# Cumulative Hit Rate; hit rate, confined to ratings above a certain threshold. Higher is better. #" + EndBgBLUE) print(
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True, sample_topN_for_userIDs=[]): sample_topN = {} self.evaluated_metrics = {} self.N = n output = {} self.recommender_data = evaluationData # sample_TopN_all = {} # sample_TopN_user = [] # # creating dictionary like: userID -> algorithms -> Top N # for userID in sample_topN_for_userIDs: # algos = {} # for algorithm, name in self.algo_comparison_set: # algos.update({name: []) # sample_TopN_all[userID] = algos # Use train-test-split dataset for RMSE and MAE scores self.recommender_algorithm.fit(self.recommender_data.GetTrainSet()) predictions = self.recommender_algorithm.test( self.recommender_data.GetTestSet()) self.evaluated_metrics["RMSE"] = RecommenderMetrics.RMSE(predictions) self.evaluated_metrics["MAE"] = RecommenderMetrics.MAE(predictions) # do only if we want top N recommendations. Compute intensive operation if (doTopN): # use Leave one out algorithm self.recommender_algorithm.fit( self.recommender_data.GetLOOCVTrainSet()) leftout_predictions = self.recommender_algorithm.test( self.recommender_data.GetLOOCVTestSet()) predictions_all_minus_train = self.recommender_algorithm.test( self.recommender_data.GetLOOCVAntiTestSet()) topN_predictions = RecommenderMetrics.GetTopN( predictions_all_minus_train, self.N, minimumRating=4.0) self.evaluated_metrics["HitRate"] = RecommenderMetrics.HitRate( topN_predictions, leftout_predictions) self.evaluated_metrics[ "CumulativeHitRate"] = RecommenderMetrics.CumulativeHitRate( topN_predictions, leftout_predictions) # self.evaluated_metrics["RatingHitRate"] = RecommenderMetrics.RatingHitRate(topN_predictions, leftout_predictions) self.evaluated_metrics[ "AverageReciprocalHitRank"] = RecommenderMetrics.AverageReciprocalHitRank( topN_predictions, leftout_predictions) # use full dataset for these metrics: UserCoverage, Diversity, Novelty trainset_full = self.recommender_data.GetFullTrainSet() self.recommender_algorithm.fit(trainset_full) predictions_all = self.recommender_algorithm.test( self.recommender_data.GetFullAntiTestSet()) topN_predictions = RecommenderMetrics.GetTopN(predictions_all, self.N) if len(sample_topN_for_userIDs) != 0: sample_topN = self.FilterTopN(topN_predictions, sample_topN_for_userIDs) self.evaluated_metrics[ "UserCoverage"] = RecommenderMetrics.UserCoverage( topN_predictions, trainset_full.n_users, ratingThreshold=4.0) # Diversity uses the similarity matrix self.evaluated_metrics["Diversity"] = RecommenderMetrics.Diversity( topN_predictions, self.recommender_data.GetSimilarities()) # Novelty uses the Popularity rankings self.evaluated_metrics["Novelty"] = RecommenderMetrics.Novelty( topN_predictions, self.recommender_data.GetPopularityRankings()) # format: {Algorithm: {evaluated metrics}} output[self.recommender_name] = {"metrics": self.evaluated_metrics} if len(sample_topN_for_userIDs) != 0: # format: {TopN: {userID: {Algorithm: [Sample_TopN movies recommended]}}} output[self.recommender_name].update({"sample_topn": sample_topN}) return output