def __evaluate():
print "Evaluation\n"
from app.algorithm.cars.info_gain.info_gain_recommender import InfoGainRecommender
context_conditions = {
'Time': ['Weekday', 'Weekend'],
'Location': ['Home', 'Cinema'],
'Companion': ['Alone', 'Family', 'Partner'],
}
result = AutoVivification()
# for context in context_conditions.keys():
# for condition in context_conditions[context]:
# print context
# print condition
# data_object = DataObject()
# recommender = InfoGainRecommender(data_object)
# recommender.run()
# recommender.filters = [(context, condition)]
# KFoldRMSE(recommender.training_data, recommender)
for context in context_conditions.keys():
for condition in context_conditions[context]:
print "Context: ", context
print "Condition: ", condition
for topN in range(1, 11):
data_object = DataObject()
recommender = InfoGainRecommender(data_object)
recommender.run()
recommender.filters = [(context, condition)]
current_results = KFold(recommender.training_data, recommender, topN=topN)
result["top-" + str(topN)]['ctx'][str(context)]['2d'] = current_results['2d']
result["top-" + str(topN)]['ctx'][str(context)][str(condition)] = current_results['ctx']
filename = "out.json"
results_to_json(result, filename)
plot_results(result)
def KFoldRMSE(data, recommender, simMeasure=sim_pearson, nNeighbors=50, topN=10, nFolds=4):
result = AutoVivification()
start_time = datetime.now()
cnt_2d = 0
cnt_ctx = 0
sum_2d = 0
sum_ctx = 0
for fold in range(nFolds):
trainSet, testSet = KFoldSplit(data, fold, nFolds)
recommender.set_training_set(trainSet)
for user in testSet.keys():
all_2d_recommendations = []
all_ctx_recommendations = []
_all_2d_recommendations = recommender.TraditionalRecommendation(user, simMeasure, nNeighbors, 9999999999)
_all_ctx_recommendations = recommender.PostFilteringRecommendation(user, simMeasure, nNeighbors, 999999999)
for rating, movie in _all_2d_recommendations:
for ctx_rating, ctx_movie in _all_ctx_recommendations:
if ctx_rating > 0 and rating > 0 and ctx_movie == movie:
all_2d_recommendations.append((rating, movie))
all_ctx_recommendations.append((ctx_rating, movie))
if len(all_2d_recommendations) != len(all_ctx_recommendations):
raise "Recs are not the same size"
for calc_rating, rec_movie in all_2d_recommendations:
if rec_movie in testSet[user].keys():
rating = testSet[user][rec_movie][0]
dif = (calc_rating - rating) ** 2
sum_2d = sum_2d + dif
cnt_2d += 1
for calc_rating, rec_movie in all_ctx_recommendations:
if rec_movie in testSet[user].keys():
rating = testSet[user][rec_movie][0]
if calc_rating >= 5:
calc_rating = 5
if calc_rating < 0:
calc_rating = 0
dif = (calc_rating - rating) ** 2
sum_ctx = sum_ctx + dif
cnt_ctx += 1
rmse_2d = sqrt(sum_2d / cnt_2d)
rmse_ctx = sqrt(sum_ctx / cnt_ctx)
print "RMSE fron traditional recommender: ", str(rmse_2d)
print "RMSE fron Contextual recommender: ", str(rmse_ctx)
print "ctn_2d", cnt_2d
print "ctn_ctx", cnt_ctx
print "sum_2d", sum_2d
print "sum_ctx", sum_ctx
print "************************************************************"
def KFold(data, recommender, simMeasure=sim_pearson, nNeighbors=50, topN=10, nFolds=4):
result = AutoVivification()
start_time = datetime.now()
for type in ['2d', 'ctx']:
# Evaluation metrics
totalPrecision = 0
totalRecall = 0
totalF1score = 0
totalHitrate = 0
for fold in range(nFolds):
trainSet, testSet = KFoldSplit(data, fold, nFolds)
evaluation = evaluateRecommender(testSet, trainSet, recommender, simMeasure=simMeasure, nNeighbors=nNeighbors, topN=topN, type=type)
totalPrecision += evaluation["Precision"]
totalRecall += evaluation["Recall"]
totalF1score += evaluation["F1-score"]
totalHitrate += evaluation["Hit-rate"]
# del (trainSet)
# del (testSet)
# Find final results
result[type]["Precision"] = totalPrecision / nFolds
result[type]["Recall"] = totalRecall / nFolds
result[type]["F1-score"] = totalF1score / nFolds
result[type]["Hit-rate"] = float(totalHitrate) / nFolds
print("Execution time: {}".format(datetime.now() - start_time))
print "******** TOPN=", str(topN), " ******"
print "2D f1Score: ", result['2d']['F1-score']
print "2D precision: ", result['2d']['Precision']
print "2D recall: ", result['2d']['Recall']
print "Contextual f2score:", result['ctx']['F1-score']
print "Contextual Precision:", result['ctx']['Precision']
print "Contextual Recall:", result['ctx']['Recall']
# plot_results(result, type='F1-score')
return result
def evaluateRecommender(testSet, trainSet, recommender, simMeasure=None, nNeighbors=None, topN=None, type='2d'):
result = AutoVivification()
recommender.set_training_set(trainSet)
# Evaluation metrics
totalPrecision = 0
totalRecall = 0
totalF1score = 0
totalHit = 0
totalRelevantItem = 0
for user in testSet.keys():
if type == '2d':
recommendation = recommender.TraditionalRecommendation(user, simMeasure, nNeighbors, topN)
else:
recommendation = recommender.PostFilteringRecommendation(user, simMeasure, nNeighbors, topN)
hit = 0
for item in testSet[user].keys():
for rating, recommended_item in recommendation:
if recommended_item == item:
hit += 1
break
precision = float(hit) / float(topN)
recall = 0 if len(testSet[user].keys()) == 0 else float(hit) / (len(testSet[user].keys()))
f1score = 0 if hit == 0 or precision + recall == 0 else float(2 * precision * recall / (precision + recall))
totalPrecision += precision
totalRecall += recall
totalF1score += f1score
totalHit += hit
result["Precision"] = float(totalPrecision / (len(testSet)))
result["Recall"] = float(totalRecall / len(testSet))
result["F1-score"] = float(totalF1score / len(testSet))
result["Hit-rate"] = float(totalHit) / len(testSet)
return result
def plot(self):
import numpy as np
import matplotlib.pyplot as plt
for context in self.metrics.keys():
labels = []
baseline_metrics = (self.metrics[context][self.metrics[context].keys()[0]]['precision'],
self.metrics[context][self.metrics[context].keys()[0]]['recall'],
self.metrics[context][self.metrics[context].keys()[0]]['f1Score'])
contextual_metrics = AutoVivification()
labels.extend(['Baseline', 'Baseline', 'Baseline'])
for condition in self.metrics[context].keys():
contextual_metrics[condition] = (self.metrics[context][condition]['precision-ctx'],
self.metrics[context][condition]['recall-ctx'],
self.metrics[context][condition]['f1Score-ctx'])
labels.extend([condition, condition, condition])
fig, ax = plt.subplots()
index = np.arange(3)
bar_width = 0.15
opacity = 0.8
rects1 = plt.bar(index, baseline_metrics, bar_width,
alpha=opacity,
color='b',
label='Baseline')
colors = {
'Weekend': 'y',
'Weekday': 'c',
'Home': 'y',
'Cinema': 'c',
'Alone': 'y',
'Partner': 'c',
'Family': '#ee1f3f',
}
i = 1
for condition in contextual_metrics.keys():
plt.bar(index + bar_width * i, contextual_metrics[condition], bar_width,
alpha=opacity,
color=colors[condition],
label=condition)
i += 1
rects = ax.patches
# Now make some labels
# labels = ["label%d" % i for i in xrange(len(rects))]
for rect, label in zip(rects, labels):
height = rect.get_height()
# ax.text(rect.get_x() + rect.get_width() / 2, height, None, ha='center', va='bottom')
# plt.xlabel('Metric')
# plt.ylabel('Value')
plt.title('Evaluation Metrics for ' + context)
plt.xticks(index + bar_width + 0.15, ('Precision', 'Recall', 'F1Score'))
import math
plt.ylim([0, 1])
plt.legend()
plt.tight_layout()
plt.show()
def plot_results(data, type=None):
import numpy as np
import matplotlib.pyplot as plt
for context in data['ctx'].keys():
labels = []
baseline_metrics = (data['ctx'][context]['2d']['Precision'], data['ctx'][context]['2d']['Recall'], data['ctx'][context]['2d']['F1-score'])
contextual_metrics = AutoVivification()
labels.extend(['Baseline', 'Baseline', 'Baseline'])
for condition in data['ctx'][context].keys():
if condition == '2d':
continue
contextual_metrics[condition] = (data['ctx'][context][condition]['Precision'],
data['ctx'][context][condition]['Recall'],
data['ctx'][context][condition]['F1-score'])
label_name = ContextConditionMappings[context][int(condition)]
labels.extend([label_name, label_name, label_name])
print baseline_metrics
print contextual_metrics
fig, ax = plt.subplots()
index = np.arange(3)
bar_width = 0.15
opacity = 0.8
rects1 = plt.bar(index, baseline_metrics, bar_width,
alpha=opacity,
color='b',
label='Baseline')
colors = {
'1': 'r',
'2': 'g',
'3': 'y',
'4': 'c',
'5': 'm',
'6': 'k',
'7': '#ee1f3f',
'8': '#123ef2'
}
i = 1
for condition in contextual_metrics.keys():
plt.bar(index + bar_width * i, contextual_metrics[condition], bar_width,
alpha=opacity,
color=colors[condition],
label=ContextConditionMappings[context][int(condition)])
i += 1
rects = ax.patches
# Now make some labels
# labels = ["label%d" % i for i in xrange(len(rects))]
for rect, label in zip(rects, labels):
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width() / 2, height, label, ha='center', va='bottom')
# plt.xlabel('Metric')
# plt.ylabel('Value')
plt.title('Evaluation Metrics for ' + ContextMappings[context] + ' context')
plt.xticks(index + bar_width + 0.15, ('Precision', 'Recall', 'F1Score'))
plt.legend()
plt.tight_layout()
plt.show()
from app.algorithm.cars.info_gain.evaluation import __evaluate
from app.algorithm.cars.info_gain.evaluation import plot_results
from app.algorithm.cars.info_gain.evaluation import results_to_json
from app.algorithm.cars.info_gain.plotter import Plotter
from app.dataset.data_object import DataObject
from app.algorithm.cars.info_gain.info_gain_recommender import InfoGainRecommender
from app.dataset.loader import AutoVivification
import argparse
context_conditions = {
'Time': ['Weekday', 'Weekend'],
'Location': ['Home', 'Cinema'],
'Companion': ['Alone', 'Family', 'Partner'],
}
result = AutoVivification()
def plot_results():
import json
with open('precisionTopNEvaluation.json', 'r') as fp:
data = json.load(fp)
plotter = Plotter(data)
plotter.plot()
plotter.plotf1Score()
def main():
plot_results()
exit()
# __evaluate()
def plot(self):
import numpy as np
import matplotlib.pyplot as plt
for context in self.metrics.keys():
labels = []
baseline_metrics = (self.metrics[context][
self.metrics[context].keys()[0]]['precision'],
self.metrics[context][
self.metrics[context].keys()[0]]['recall'],
self.metrics[context]
[self.metrics[context].keys()[0]]['f1Score'])
contextual_metrics = AutoVivification()
labels.extend(['Baseline', 'Baseline', 'Baseline'])
for condition in self.metrics[context].keys():
contextual_metrics[condition] = (
self.metrics[context][condition]['precision-ctx'],
self.metrics[context][condition]['recall-ctx'],
self.metrics[context][condition]['f1Score-ctx'])
labels.extend([condition, condition, condition])
fig, ax = plt.subplots()
index = np.arange(3)
bar_width = 0.15
opacity = 0.8
rects1 = plt.bar(index,
baseline_metrics,
bar_width,
alpha=opacity,
color='b',
label='Baseline')
colors = {
'Weekend': 'y',
'Weekday': 'c',
'Home': 'y',
'Cinema': 'c',
'Alone': 'y',
'Partner': 'c',
'Family': '#ee1f3f',
}
i = 1
for condition in contextual_metrics.keys():
plt.bar(index + bar_width * i,
contextual_metrics[condition],
bar_width,
alpha=opacity,
color=colors[condition],
label=condition)
i += 1
rects = ax.patches
# Now make some labels
# labels = ["label%d" % i for i in xrange(len(rects))]
for rect, label in zip(rects, labels):
height = rect.get_height()
# ax.text(rect.get_x() + rect.get_width() / 2, height, None, ha='center', va='bottom')
# plt.xlabel('Metric')
# plt.ylabel('Value')
plt.title('Evaluation Metrics for ' + context)
plt.xticks(index + bar_width + 0.15,
('Precision', 'Recall', 'F1Score'))
import math
plt.ylim([0, 1])
plt.legend()
plt.tight_layout()
plt.show()