def add_result(self, ranking, sample):
predicted = RecommendationResult(dict.fromkeys(ranking, 1))
real = RecommendationResult(sample)
evaluation = Evaluation(predicted, real, self.repository_size)
self.precision.append(evaluation.run(Precision()))
self.recall.append(evaluation.run(Recall()))
self.fpr.append(evaluation.run(FPR()))
self.f05.append(evaluation.run(F_score(0.5)))
self.mcc.append(evaluation.run(MCC()))
def add_result(self, ranking, sample):
for size in self.accuracy.keys():
predicted = RecommendationResult(dict.fromkeys(ranking[:size], 1))
real = RecommendationResult(sample)
evaluation = Evaluation(predicted, real, self.repository_size)
self.accuracy[size].append(evaluation.run(Accuracy()))
self.precision[size].append(evaluation.run(Precision()))
self.recall[size].append(evaluation.run(Recall()))
self.f1[size].append(evaluation.run(F_score(1)))
self.f05[size].append(evaluation.run(F_score(0.5)))
class RecommendationResultTests(unittest2.TestCase):
@classmethod
def setUpClass(self):
self.result = RecommendationResult({"gimp":1.5,"inkscape":3.0,"eog":1})
def test_str(self):
string = "\n 0: inkscape\n 1: gimp\n 2: eog\n"
self.assertEqual(self.result.__str__(),string)
def test_get_prediction(self):
prediction = [("inkscape",3.0),("gimp",1.5),("eog",1)]
self.assertEqual(self.result.get_prediction(),prediction)
def add_result(self, ranking, sample):
for size in self.thresholds:
recommendation = ranking[:size]
self.recommended[size] = self.recommended[size].union(
recommendation)
predicted = RecommendationResult(dict.fromkeys(recommendation, 1))
real = RecommendationResult(sample)
evaluation = Evaluation(predicted, real, self.repository_size)
self.precision[size].append(evaluation.run(Precision()))
self.recall[size].append(evaluation.run(Recall()))
self.f05[size].append(evaluation.run(F_score(0.5)))
self.fpr[size].append(evaluation.run(FPR()))
def iterate(self, params, rep, n):
if params['name'].startswith("content"):
item_score = dict.fromkeys(self.user.pkg_profile, 1)
# Prepare partition
sample = {}
for i in range(self.sample_size):
key = random.choice(item_score.keys())
sample[key] = item_score.pop(key)
# Get full recommendation
user = User(item_score)
recommendation = self.rec.get_recommendation(user, self.repo_size)
# Write recall log
recall_file = "results/content/recall/%s-%s-%.2f-%d" % \
(params['strategy'], params[
'weight'], params['sample'], n)
output = open(recall_file, 'w')
output.write("# weight=%s\n" % params['weight'])
output.write("# strategy=%s\n" % params['strategy'])
output.write("# sample=%f\n" % params['sample'])
output.write("\n%d %d %d\n" %
(self.repo_size, len(item_score), self.sample_size))
notfound = []
ranks = []
for pkg in sample.keys():
if pkg in recommendation.ranking:
ranks.append(recommendation.ranking.index(pkg))
else:
notfound.append(pkg)
for r in sorted(ranks):
output.write(str(r) + "\n")
if notfound:
output.write("Out of recommendation:\n")
for pkg in notfound:
output.write(pkg + "\n")
output.close()
# Plot metrics summary
accuracy = []
precision = []
recall = []
f1 = []
g = Gnuplot.Gnuplot()
g('set style data lines')
g.xlabel('Recommendation size')
for size in range(1, len(recommendation.ranking) + 1, 100):
predicted = RecommendationResult(
dict.fromkeys(recommendation.ranking[:size], 1))
real = RecommendationResult(sample)
evaluation = Evaluation(predicted, real, self.repo_size)
accuracy.append([size, evaluation.run(Accuracy())])
precision.append([size, evaluation.run(Precision())])
recall.append([size, evaluation.run(Recall())])
f1.append([size, evaluation.run(F1())])
g.plot(Gnuplot.Data(accuracy, title="Accuracy"),
Gnuplot.Data(precision, title="Precision"),
Gnuplot.Data(recall, title="Recall"),
Gnuplot.Data(f1, title="F1"))
g.hardcopy(recall_file + "-plot.ps", enhanced=1, color=1)
# Iteration log
result = {'iteration': n,
'weight': params['weight'],
'strategy': params['strategy'],
'accuracy': accuracy[20],
'precision': precision[20],
'recall:': recall[20],
'f1': f1[20]}
return result
# Fill sample profile
profile_len = len(user.pkg_profile)
item_score = {}
for pkg in user.pkg_profile:
item_score[pkg] = user.item_score[pkg]
sample = {}
sample_size = int(profile_len * 0.9)
for i in range(sample_size):
key = random.choice(item_score.keys())
sample[key] = item_score.pop(key)
iteration_user = User(item_score)
recommendation = rec.get_recommendation(
iteration_user, repo_size)
if hasattr(recommendation, "ranking"):
ranking = recommendation.ranking
real = RecommendationResult(sample)
predicted_10 = RecommendationResult(
dict.fromkeys(ranking[:10], 1))
evaluation = Evaluation(predicted_10, real, repo_size)
p_10.append(evaluation.run(Precision()))
predicted_100 = RecommendationResult(
dict.fromkeys(ranking[:100], 1))
evaluation = Evaluation(predicted_100, real, repo_size)
f05_100.append(evaluation.run(F_score(0.5)))
c_10[k][size] = c_10[k][size].union(
recommendation.ranking[:10])
c_100[k][size] = c_100[k][size].union(
recommendation.ranking[:100])
# save summary
if p_10:
p_10_summary[k][size].append(numpy.mean(p_10))
def setUpClass(self):
self.result = RecommendationResult({"gimp":1.5,"inkscape":3.0,"eog":1})
