def Evaluate(self, evaluationData, doTopN, n=10, verbose=True):
metrics = {}
# Compute accuracy
if (verbose):
print("Evaluating accuracy...")
self.algorithm.fit(evaluationData.GetTrainSet())
predictions = self.algorithm.test(evaluationData.GetTestSet())
metrics["RMSE"] = RecommenderMetrics.RMSE(predictions)
metrics["MAE"] = RecommenderMetrics.MAE(predictions)
if (doTopN):
# Evaluate top-10 with Leave One Out testing
if (verbose):
print("Evaluating top-N with leave-one-out...")
self.algorithm.fit(evaluationData.GetLOOCVTrainSet())
leftOutPredictions = self.algorithm.test(
evaluationData.GetLOOCVTestSet())
# Build predictions for all ratings not in the training set
allPredictions = self.algorithm.test(
evaluationData.GetLOOCVAntiTestSet())
# Compute top 10 recs for each user
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
if (verbose):
print("Computing hit-rate and rank metrics...")
# See how often we recommended a movie the user actually rated
metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted,
leftOutPredictions)
# See how often we recommended a movie the user actually liked
metrics["cHR"] = RecommenderMetrics.CumulativeHitRate(
topNPredicted, leftOutPredictions)
# Compute ARHR
metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank(
topNPredicted, leftOutPredictions)
#Evaluate properties of recommendations on full training set
if (verbose):
print("Computing recommendations with full data set...")
self.algorithm.fit(evaluationData.GetFullTrainSet())
allPredictions = self.algorithm.test(
evaluationData.GetFullAntiTestSet())
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
if (verbose):
print("Analyzing coverage, diversity, and novelty...")
# Print user coverage with a minimum predicted rating of 4.0:
metrics["Coverage"] = RecommenderMetrics.UserCoverage(
topNPredicted,
evaluationData.GetFullTrainSet().n_users,
ratingThreshold=4.0)
# Measure diversity of recommendations:
metrics["Diversity"] = RecommenderMetrics.Diversity(
topNPredicted, evaluationData.GetSimilarities())
# Measure novelty (average popularity rank of recommendations):
metrics["Novelty"] = RecommenderMetrics.Novelty(
topNPredicted, evaluationData.GetPopularityRankings())
if (verbose):
print("Analysis complete.")
return metrics
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True):
metrics = {}
# Compute accuracy
if (verbose):
print("Evaluating accuracy...")
self.algorithm.fit(evaluationData.GetTrainSet())
predictions = self.algorithm.test(evaluationData.GetTestSet())
metrics["RMSE"] = RecommenderMetrics.RMSE(predictions)
metrics["MAE"] = RecommenderMetrics.MAE(predictions)
if (doTopN):
# Evaluate top-10 with Leave One Out testing
if (verbose):
print("leave-one-out法でtop-Nを計算...")
self.algorithm.fit(evaluationData.GetLOOCVTrainSet())
leftOutPredictions = self.algorithm.test(
evaluationData.GetLOOCVTestSet())
# 訓練データではない方で
allPredictions = self.algorithm.test(
evaluationData.GetLOOCVAntiTestSet())
# 各ユーザーに10個の表示を行う
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
if (verbose):
print("Computing hit-rate and rank metrics...")
# HitRate: どのくらいの頻度で、ユーザーが評価しうているかの指標
metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted,
leftOutPredictions)
# cHitRate: ユーザーが実際にlikedしたかを確認する
metrics["cHR"] = RecommenderMetrics.CumulativeHitRate(
topNPredicted, leftOutPredictions)
# ARHR
metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank(
topNPredicted, leftOutPredictions)
#すべての訓練データセットでのレコメンデーションの特徴量を評価する
if (verbose):
print("すべてのデータセットでレコメンドの計算をしています...")
self.algorithm.fit(evaluationData.GetFullTrainSet())
allPredictions = self.algorithm.test(
evaluationData.GetFullAntiTestSet())
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
if (verbose):
print("Analyzing coverage, diversity, and novelty...")
# 4.0を閾値として表示する
metrics["Coverage"] = RecommenderMetrics.UserCoverage(
topNPredicted,
evaluationData.GetFullTrainSet().n_users,
ratingThreshold=4.0)
# レコメンドの多様性の測定
metrics["Diversity"] = RecommenderMetrics.Diversity(
topNPredicted, evaluationData.GetSimilarities())
# レコメンドされた人気の高い評価の平均値を計算する
metrics["Novelty"] = RecommenderMetrics.Novelty(
topNPredicted, evaluationData.GetPopularityRankings())
if (verbose):
print("Analysis complete.")
return metrics
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True):
metrics = {}
# Compute accuracy
if (verbose):
print("Evaluating: ", self.name)
self.algorithm.fit(evaluationData.GetTrainSet())
predictions = self.algorithm.test(evaluationData.GetTestSet())
metrics["RMSE"] = RecommenderMetrics.RMSE(predictions)
metrics["MAE"] = RecommenderMetrics.MAE(predictions)
if (doTopN):
# Evaluate top-10 with Leave One Out testing
self.algorithm.fit(evaluationData.GetLOOCVTrainSet())
leftOutPredictions = self.algorithm.test(
evaluationData.GetLOOCVTestSet())
# Build predictions for all ratings not in the training set
allPredictions = self.algorithm.test(
evaluationData.GetLOOCVAntiTestSet())
# Compute top 10 recs for each user
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
# See how often we recommended a movie the user actually rated
metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted,
leftOutPredictions)
#Evaluate properties of recommendations on full training set
self.algorithm.fit(evaluationData.GetFullTrainSet())
allPredictions = self.algorithm.test(
evaluationData.GetFullAntiTestSet())
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
# Print user coverage with a minimum predicted rating of 6.0:
metrics["Coverage"] = RecommenderMetrics.UserCoverage(
topNPredicted,
evaluationData.GetFullTrainSet().n_users,
ratingThreshold=7.0)
# Measure novelty (average popularity rank of recommendations):
metrics["Novelty"] = RecommenderMetrics.Novelty(
topNPredicted, evaluationData.GetPopularityRankings())
return metrics
def get_hit_rates(trainSet, testSet, simsMatrix, num_nearest_neighbors=10):
# Build up dict to lists of (int(movieID), predictedrating) pairs
topN = defaultdict(list)
k = num_nearest_neighbors
for uiid in range(trainSet.n_users):
# Get top N similar users to this one
similarityRow = simsMatrix[uiid]
similarUsers = []
for innerID, score in enumerate(similarityRow):
if (innerID != uiid):
similarUsers.append((innerID, score))
kNeighbors = heapq.nlargest(k, similarUsers, key=lambda t: t[1])
# Get the stuff they rated, and add up ratings for each item, weighted by user similarity
candidates = defaultdict(float)
for similarUser in kNeighbors:
innerID = similarUser[0]
userSimilarityScore = similarUser[1]
theirRatings = trainSet.ur[innerID]
for rating in theirRatings:
candidates[rating[0]] += (rating[1] /
5.0) * userSimilarityScore
# Build a dictionary of stuff the user has already seen
watched = {}
for itemID, rating in trainSet.ur[uiid]:
watched[itemID] = 1
# Get top-rated items from similar users:
pos = 0
for itemID, ratingSum in sorted(candidates.items(),
key=itemgetter(1),
reverse=True):
if not itemID in watched:
movieID = trainSet.to_raw_iid(itemID)
topN[int(trainSet.to_raw_uid(uiid))].append(
(int(movieID), 0.0))
pos += 1
if (pos > 40):
break
HR = RecommenderMetrics.HitRate(topN, testSet)
CHR = RecommenderMetrics.CumulativeHitRate(topN, testSet, ratingCutoff=4.0)
RHR = RecommenderMetrics.RatingHitRate(topN, testSet)
ARHR = RecommenderMetrics.AverageReciprocalHitRank(topN, testSet)
return HR, CHR, RHR, ARHR
# Predicts ratings for left-out ratings only
print("Predict ratings for left-out set...")
leftOutPredictions = algo.test(testSet)
# Build predictions for all ratings not in the training set
print("Predict all missing ratings...")
bigTestSet = trainSet.build_anti_testset()
allPredictions = algo.test(bigTestSet)
# Compute top 10 recs for each user
print("Compute top 10 recs per user...")
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n=10)
# See how often we recommended a movie the user actually rated
print("\nHit Rate: ", RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions))
# Break down hit rate by rating value
print("\nrHR (Hit Rate by Rating value): ")
RecommenderMetrics.RatingHitRate(topNPredicted, leftOutPredictions)
# See how often we recommended a movie the user actually liked
print("\ncHR (Cumulative Hit Rate, rating >= 4): ", RecommenderMetrics.CumulativeHitRate(topNPredicted, leftOutPredictions, 4.0))
# Compute ARHR
print("\nARHR (Average Reciprocal Hit Rank): ", RecommenderMetrics.AverageReciprocalHitRank(topNPredicted, leftOutPredictions))
print("\nComputing complete recommendations, no hold outs...")
algo.fit(fullTrainSet)
bigTestSet = fullTrainSet.build_anti_testset()
allPredictions = algo.test(bigTestSet)
def Evaluate(self, evaluationData, doTopN, n=10, verbose=True, load=False):
metrics = {}
# Compute accuracy
if (verbose):
print("Evaluating accuracy...")
pr = None
alg = None
if load:
file_name = self.name + '_acc'
pr, alg, _ = MyDump.Load(file_name, 1)
if pr == None or alg == None:
self.algorithm.fit(evaluationData.GetTrainSet()) # 75%
predictions = self.algorithm.test(evaluationData.GetTestSet())
if load:
MyDump.Save(file_name, predictions, self.algorithm, None, 1)
else:
self.algorithm = alg
predictions = pr
metrics["RMSE"] = RecommenderMetrics.RMSE(predictions)
metrics["MAE"] = RecommenderMetrics.MAE(predictions)
# if load:
# MyDump.Save(file_name, predictions, self.algorithm, None, 1)
if (doTopN):
# if False:
# Evaluate top-10 with Leave One Out testing
if (verbose):
print("Evaluating top-N with leave-one-out...")
pr_top = None
alg_top = None
if load:
file_name = self.name + '_top' + str(n)
pr_top, alg_top, _ = MyDump.Load(file_name, 1)
if pr_top == None or alg_top == None:
self.algorithm.fit(evaluationData.GetLOOCVTrainSet())
leftOutPredictions = self.algorithm.test(
evaluationData.GetLOOCVTestSet())
# Build predictions for all ratings not in the training set
allPredictions = self.algorithm.test(
evaluationData.GetLOOCVAntiTestSet())
# Compute top 10 recs for each user
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
pr_data = {
'leftOutPredictions': leftOutPredictions,
'topNPredicted': topNPredicted
}
if load:
MyDump.Save(file_name, pr_data, self.algorithm, None, 1)
else:
self.algorithm = alg_top
leftOutPredictions = pr_top['leftOutPredictions']
topNPredicted = pr_top['topNPredicted']
if (verbose):
print("Computing hit-rate and rank metrics...")
# See how often we recommended a movie the user actually rated
metrics["HR"] = RecommenderMetrics.HitRate(topNPredicted,
leftOutPredictions)
# See how often we recommended a movie the user actually liked
metrics["cHR"] = RecommenderMetrics.CumulativeHitRate(
topNPredicted, leftOutPredictions)
# Compute ARHR
metrics["ARHR"] = RecommenderMetrics.AverageReciprocalHitRank(
topNPredicted, leftOutPredictions)
#Evaluate properties of recommendations on full training set
if (verbose):
print("Computing recommendations with full data set...")
pr_top = None
alg_top = None
if load:
file_name = self.name + '_fulldata'
pr_top, alg_top, _ = MyDump.Load(file_name, 1)
if pr_top == None or alg_top == None:
self.algorithm.fit(evaluationData.GetFullTrainSet())
allPredictions = self.algorithm.test(
evaluationData.GetFullAntiTestSet())
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n)
pr_data = {
'allPredictions': allPredictions,
'topNPredicted': topNPredicted
}
if load:
MyDump.Save(file_name, pr_data, self.algorithm, None, 1)
else:
self.algorithm = alg_top
allPredictions = pr_top['allPredictions']
topNPredicted = pr_top['topNPredicted']
if (verbose):
print("Analyzing coverage, diversity, and novelty...")
# Print user coverage with a minimum predicted rating of 4.0:
metrics["Coverage"] = RecommenderMetrics.UserCoverage(
topNPredicted,
evaluationData.GetFullTrainSet().n_users,
ratingThreshold=4.0)
# Measure diversity of recommendations:
metrics["Diversity"] = RecommenderMetrics.Diversity(
topNPredicted, evaluationData.GetSimilarities())
# Measure novelty (average popularity rank of recommendations):
metrics["Novelty"] = RecommenderMetrics.Novelty(
topNPredicted, evaluationData.GetPopularityRankings())
if (verbose):
print("Analysis complete.")
# SaveAlg()
return metrics
kNeighbors = heapq.nlargest(k, similarUsers, key=lambda t: t[1])
# Get the stuff they rated, and add up ratings for each item, weighted by user similarity
candidates = defaultdict(float)
for similarUser in kNeighbors:
innerID = similarUser[0]
userSimilarityScore = similarUser[1]
theirRatings = trainSet.ur[innerID]
for rating in theirRatings:
candidates[rating[0]] += (rating[1] / 5.0) * userSimilarityScore
# Build a dictionary of stuff the user has already seen
watched = {}
for itemID, rating in trainSet.ur[uiid]:
watched[itemID] = 1
# Get top-rated items from similar users:
pos = 0
for itemID, ratingSum in sorted(candidates.items(), key=itemgetter(1), reverse=True):
if not itemID in watched:
movieID = trainSet.to_raw_iid(itemID)
topN[int(trainSet.to_raw_uid(uiid))].append( (int(movieID), 0.0) )
pos += 1
if (pos > 40):
break
# Measure
print("HR", RecommenderMetrics.HitRate(topN, leftOutTestSet))
# Build predictions for all ratings not in the training set
print(BeginGREEN + "Predict all missing ratings..." + EndGREEN)
bigTestSet = trainSet.build_anti_testset()
allPredictions = algo.test(bigTestSet)
# Compute top 10 recs for each user
print(BeginGREEN + "Compute top 10 recs per user..." + EndGREEN)
topNPredicted = RecommenderMetrics.GetTopN(allPredictions, n=10)
print("\n")
# See how often we recommended a movie the user actually rated
print(
BeginBgBLUE +
"# Hit Rate; how often we are able to recommend a left-out rating. Higher is better. #"
+ EndBgBLUE)
print(BeginBLUE + "Hit Rate: " + EndBLUE, BeginYELLO + "",
RecommenderMetrics.HitRate(topNPredicted, leftOutPredictions),
"" + EndYELLO)
print("\n")
# Break down hit rate by rating value
print(BeginBgBLUE + "# rating Hit Rate #" + EndBgBLUE)
print(BeginBLUE + "rHR (Hit Rate by Rating value): " + EndBLUE)
RecommenderMetrics.RatingHitRate(topNPredicted, leftOutPredictions)
print("\n")
# See how often we recommended a movie the user actually liked
print(
BeginBgBLUE +
"# Cumulative Hit Rate; hit rate, confined to ratings above a certain threshold. Higher is better. #"
+ EndBgBLUE)
print(
def Evaluate(self,
evaluationData,
doTopN,
n=10,
verbose=True,
sample_topN_for_userIDs=[]):
sample_topN = {}
self.evaluated_metrics = {}
self.N = n
output = {}
self.recommender_data = evaluationData
# sample_TopN_all = {}
# sample_TopN_user = []
# # creating dictionary like: userID -> algorithms -> Top N
# for userID in sample_topN_for_userIDs:
# algos = {}
# for algorithm, name in self.algo_comparison_set:
# algos.update({name: [])
# sample_TopN_all[userID] = algos
# Use train-test-split dataset for RMSE and MAE scores
self.recommender_algorithm.fit(self.recommender_data.GetTrainSet())
predictions = self.recommender_algorithm.test(
self.recommender_data.GetTestSet())
self.evaluated_metrics["RMSE"] = RecommenderMetrics.RMSE(predictions)
self.evaluated_metrics["MAE"] = RecommenderMetrics.MAE(predictions)
# do only if we want top N recommendations. Compute intensive operation
if (doTopN):
# use Leave one out algorithm
self.recommender_algorithm.fit(
self.recommender_data.GetLOOCVTrainSet())
leftout_predictions = self.recommender_algorithm.test(
self.recommender_data.GetLOOCVTestSet())
predictions_all_minus_train = self.recommender_algorithm.test(
self.recommender_data.GetLOOCVAntiTestSet())
topN_predictions = RecommenderMetrics.GetTopN(
predictions_all_minus_train, self.N, minimumRating=4.0)
self.evaluated_metrics["HitRate"] = RecommenderMetrics.HitRate(
topN_predictions, leftout_predictions)
self.evaluated_metrics[
"CumulativeHitRate"] = RecommenderMetrics.CumulativeHitRate(
topN_predictions, leftout_predictions)
# self.evaluated_metrics["RatingHitRate"] = RecommenderMetrics.RatingHitRate(topN_predictions, leftout_predictions)
self.evaluated_metrics[
"AverageReciprocalHitRank"] = RecommenderMetrics.AverageReciprocalHitRank(
topN_predictions, leftout_predictions)
# use full dataset for these metrics: UserCoverage, Diversity, Novelty
trainset_full = self.recommender_data.GetFullTrainSet()
self.recommender_algorithm.fit(trainset_full)
predictions_all = self.recommender_algorithm.test(
self.recommender_data.GetFullAntiTestSet())
topN_predictions = RecommenderMetrics.GetTopN(predictions_all, self.N)
if len(sample_topN_for_userIDs) != 0:
sample_topN = self.FilterTopN(topN_predictions,
sample_topN_for_userIDs)
self.evaluated_metrics[
"UserCoverage"] = RecommenderMetrics.UserCoverage(
topN_predictions, trainset_full.n_users, ratingThreshold=4.0)
# Diversity uses the similarity matrix
self.evaluated_metrics["Diversity"] = RecommenderMetrics.Diversity(
topN_predictions, self.recommender_data.GetSimilarities())
# Novelty uses the Popularity rankings
self.evaluated_metrics["Novelty"] = RecommenderMetrics.Novelty(
topN_predictions, self.recommender_data.GetPopularityRankings())
# format: {Algorithm: {evaluated metrics}}
output[self.recommender_name] = {"metrics": self.evaluated_metrics}
if len(sample_topN_for_userIDs) != 0:
# format: {TopN: {userID: {Algorithm: [Sample_TopN movies recommended]}}}
output[self.recommender_name].update({"sample_topn": sample_topN})
return output