def evaluateRecommendations(self,
URM_test,
at=5,
minRatingsPerUser=1,
exclude_seen=True,
filterCustomItems=np.array([], dtype=np.int),
filterCustomUsers=np.array([], dtype=np.int)):
"""
Speed info:
- Sparse weighgs: batch mode is 2x faster than sequential
- Dense weighgts: batch and sequential speed are equivalent
:param URM_test: URM to be used for testing
:param at: 5 Length of the recommended items
:param minRatingsPerUser: 1 Users with less than this number of interactions will not be evaluated
:param exclude_seen: True Whether to remove already seen items from the recommended items
:param mode: 'sequential', 'parallel', 'batch'
:param filterTopPop: False or decimal number Percentage of items to be removed from recommended list and testing interactions
:param filterCustomItems: Array, default empty Items ID to NOT take into account when recommending
:param filterCustomUsers: Array, default empty Users ID to NOT take into account when recommending
:return:
"""
import warnings
warnings.warn(
"DEPRECATED! Use Base.Evaluation.SequentialEvaluator.evaluateRecommendations()",
DeprecationWarning)
from Base.Evaluation.Evaluator import SequentialEvaluator
evaluator = SequentialEvaluator(URM_test, [at],
exclude_seen=exclude_seen,
minRatingsPerUser=minRatingsPerUser,
ignore_items=filterCustomItems,
ignore_users=filterCustomUsers)
results_run, results_run_string = evaluator.evaluateRecommender(self)
results_run = results_run[at]
results_run_lowercase = {}
for key in results_run.keys():
results_run_lowercase[key.lower()] = results_run[key]
return results_run_lowercase
"lambda_j": [0.9408725374123923, 0.9408725374123923],
"num_factors": [395, 391, 95],
'alphaP3': [0.7100641282565131, 1.2827139967773968],
'alphaRP3': [0.8182264529058118, 0.8182264529058118],
'betaRP': [0.3775651302605211, 0.3775651302605211],
'l1_ratio': l1_ratio,
'epochs': 1,
"weights_to_dweights": -1})
print("TEST")
print("Starting Evaluations...")
# to indicate if plotting for lenght or for pop
results_run, results_run_string, target_recommendations = evaluator.evaluateRecommender(recommender,
plot_stats=True,
onPop=onPop)
print("Algorithm: {}, results: \n{}".format([rec.__class__ for rec in recommender.recommender_list],
results_run_string))
logFile.write("Algorithm: {}, results: \n{}\n".format(recommender.__class__, results_run_string))
logFile.flush()
if not evaluate_algorithm:
target_playlist = dataReader.get_target_playlist()
md.assign_recomendations_to_correct_playlist(target_playlist, target_recommendations)
md.make_CSV_file(target_playlist, filename)
print('File {} created!'.format(filename))
except Exception as e:
for x in only_sum_equal_1:
print("-------------------")
print("---weights = {}---".format(x))
print("-------------------")
hybridRecommender_scores.fit(weights=x)
# print("-------------------")
# print("---Hybrid fitted---")
# print("-------------------")
print("-------------------")
print("-Hybrid Evaluation-")
print("-------------------")
dict, _ = evaluator_test.evaluateRecommender(hybridRecommender_scores)
writeLog("---weights = {}---".format(x), output_file)
writeLog("--- Parameters : {} ".format(dict), output_file)
print(dict)
# target_data = pd.read_csv('data/target_playlists.csv')
#
# print("--------------------------")
# print("------Recommendation------")
# print("--------------------------")
# ws.write_submission(target_data, , 'output/submission.csv', at=10)
topK_list = list(range(1, 800, 10))
alpha_list = list(np.linspace(0.001, 2.0, 500)) # range(0, 2)
beta_list = list(np.linspace(0.001, 2.0, 500))
total_permutations = [
x for x in list(itertools.product(topK_list, alpha_list, beta_list))
]
shuffle(total_permutations)
iter = 0
start_time = time.time()
for topK, alpha, beta in total_permutations:
if time.time() - start_time > 60 * 60 * 5:
# after 5 hours stop it!
continue
if iter > 30:
continue
recommender.fit(**{"topK": topK, 'alpha': alpha, 'beta': beta})
results_run, results_run_string, target_recommendations = evaluator.evaluateRecommender(
recommender)
print("Rp3beta params topK: {}, alpha: {}, beta: {}, MAP: {}".format(
topK, alpha, beta, results_run[10]['MAP']))
gc.collect()
iter += 1
for recommender_class in recommender_list:
try:
print("Algorithm: {}".format(recommender_class))
recommender = recommender_class(URM_train)
item = ItemKNNCFRecommender(URM_train)
W_sparse_CF = item.fit(topK=800,
shrink=8,
similarity='cosine',
normalize=True)
CFW_weithing = CFW_D_Similarity_Linalg(URM_train, ICM_Art,
W_sparse_CF)
CFW_weithing.fit(topK=8, loss_tolerance=1e-04, add_zeros_quota=0)
recommender.fit(CFW=CFW_weithing)
results_run, results_run_string = evaluator.evaluateRecommender(
recommender)
print("Algorithm: {}, results: \n{}".format(
recommender.__class__, results_run_string))
logFile.write("Algorithm: {}, results: \n{}\n".format(
recommender.__class__, results_run_string))
logFile.flush()
except Exception as e:
traceback.print_exc()
logFile.write("Algorithm: {} - Exception: {}\n".format(
recommender_class, str(e)))
logFile.flush() # -*- coding: utf-8 -*-
def run():
evaluate_algorithm = True
delete_old_computations = False
slim_after_hybrid = False
# delete_previous_intermediate_computations()
# if not evaluate_algorithm:
# delete_previous_intermediate_computations()
# else:
# print("ATTENTION: old intermediate computations kept, pay attention if running with all_train")
# delete_previous_intermediate_computations()
filename = "hybrid_ICB_ICF_UCF_IALS_SLIM_ELASTIC_local_081962.csv"
dataReader = RS_Data_Loader(all_train=not evaluate_algorithm)
URM_train = dataReader.get_URM_train()
URM_PageRank_train = dataReader.get_page_rank_URM()
URM_validation = dataReader.get_URM_validation()
URM_test = dataReader.get_URM_test()
ICM = dataReader.get_ICM()
UCM_tfidf = dataReader.get_tfidf_artists()
# _ = dataReader.get_tfidf_album()
recommender_list1 = [
# Random,
# TopPop,
ItemKNNCBFRecommender,
# UserKNNCBRecommender,
ItemKNNCFRecommender,
UserKNNCFRecommender,
# P3alphaRecommender,
# RP3betaRecommender,
# MatrixFactorization_BPR_Cython,
# MatrixFactorization_FunkSVD_Cython,
IALS_numpy,
SLIM_BPR_Cython,
# ItemKNNCFRecommenderFAKESLIM,
# PureSVDRecommender,
SLIMElasticNetRecommender
]
from Base.Evaluation.Evaluator import SequentialEvaluator
evaluator = SequentialEvaluator(URM_test, URM_train, exclude_seen=True)
output_root_path = "result_experiments/"
# If directory does not exist, create
if not os.path.exists(output_root_path):
os.makedirs(output_root_path)
logFile = open(output_root_path + "result_all_algorithms.txt", "a")
try:
recommender_class = HybridRecommender
print("Algorithm: {}".format(recommender_class))
'''
Our optimal run
'''
recommender_list = recommender_list1 # + recommender_list2 # + recommender_list3
onPop = False
# On pop it used to choose if have dynamic weights for
recommender = recommender_class(URM_train,
ICM,
recommender_list,
URM_PageRank_train=URM_PageRank_train,
dynamic=False,
UCM_train=UCM_tfidf,
URM_validation=URM_validation,
onPop=onPop)
recommender.fit(
**{
"topK": [10, 181, 82, -1, 761, 490],
"shrink": [8, 0, 3, -1, -1, -1],
"pop": [280],
"weights": [
0.47412263345597117, 1.3864620551711606,
0.6224999770898935, 1.5498327677561246, 0.1993692779443738,
2.113324096784624
],
"final_weights": [1, 1],
"force_compute_sim":
False, # not evaluate_algorithm,
"feature_weighting_index":
0,
"epochs":
150,
'lambda_i': [0.0],
'lambda_j': [1.0153577332223556e-08],
'SLIM_lr': [0.1],
'alphaP3': [0.4121720883248633],
'alphaRP3': [0.8582865731462926],
'betaRP': [0.2814208416833668],
'l1_ratio':
3.020408163265306e-06,
'alpha':
0.0014681984611695231,
'tfidf': [True],
"weights_to_dweights":
-1,
"IALS_num_factors":
290,
"IALS_reg":
0.001,
"IALS_iters":
6,
"IALS_scaling":
'log',
"IALS_alpha":
40,
"filter_top_pop_len":
0
})
print("TEST")
print("Starting Evaluations...")
# to indicate if plotting for lenght or for pop
results_run, results_run_string, target_recommendations = evaluator.evaluateRecommender(
recommender, plot_stats=False, onPop=onPop)
print("Algorithm: {}, results: \n{}".format(
[rec.RECOMMENDER_NAME for rec in recommender.recommender_list],
results_run_string))
logFile.write("Algorithm: {}, results: \n{} time: {}".format(
[rec.RECOMMENDER_NAME for rec in recommender.recommender_list],
results_run_string, time.time()))
logFile.flush()
if not evaluate_algorithm:
target_playlist = dataReader.get_target_playlist()
md.assign_recomendations_to_correct_playlist(
target_playlist, target_recommendations)
md.make_CSV_file(target_playlist, filename)
print('File {} created!'.format(filename))
except Exception as e:
traceback.print_exc()
logFile.write("Algorithm: {} - Exception: {}\n".format(
recommender_class, str(e)))
logFile.flush()
os.makedirs(output_root_path)
logFile = open(output_root_path + "Hybrid_GridSearch.txt", "a")
# -------------------------------
# metric to optimize
# -------------------------------
metric_to_optimize = "map"
map_list = []
best_dict = {
"MAP": 0,
"WEIGHTS": [0, 0, 0],
}
i = 0
for x in only_sum_equal_1:
i += 1
print("---- {} \%".format(i / len(only_sum_equal_1)))
hybrid.fit(weights=x)
res_dict = evaluator_test.evaluateRecommender(hybrid)
map_list.append(res_dict[0][10]["MAP"])
if res_dict[0][10]["MAP"] > best_dict["MAP"]:
best_dict["MAP"] = res_dict[0][10]["MAP"]
best_dict["WEIGHTS"] = x
print("New best config : {}".format(best_dict))
if (logFile != None):
logFile.write("Best config: {}, Results {}\n".format(best_dict))
logFile.flush()
print("Best config : {}".format(best_dict))
from Base.Evaluation.Evaluator import SequentialEvaluator
from SLIM_BPR.Cython.SLIM_BPR_Cython import SLIM_BPR_Cython
from refactored_code.utils import load_random_urms, load_icm
if __name__ == '__main__':
urm_train, urm_test = load_random_urms()
icm = load_icm(0.7, 0.3, 0.5)
slim = SLIM_BPR_Cython(urm_train, positive_threshold=0,
symmetric=True).fit(epochs=10,
topK=300,
batch_size=500,
sgd_mode='adagrad',
learning_rate=1e-4)
evaluator_MF = SequentialEvaluator(urm_test, cutoff_list=[10])
print(evaluator_MF.evaluateRecommender(slim))
def run():
evaluate_algorithm = False
delete_old_computations = False
slim_after_hybrid = False
# delete_previous_intermediate_computations()
# if not evaluate_algorithm:
# delete_previous_intermediate_computations()
# else:
# print("ATTENTION: old intermediate computations kept, pay attention if running with all_train")
# delete_previous_intermediate_computations()
filename = "hybrid_ICB_ICF_UCF_SLIM_ELASTIC_local_08052.csv"
dataReader = RS_Data_Loader(all_train=not evaluate_algorithm)
URM_train = dataReader.get_URM_train()
URM_PageRank_train = dataReader.get_page_rank_URM()
URM_validation = dataReader.get_URM_validation()
URM_test = dataReader.get_URM_test()
ICM = dataReader.get_ICM()
UCM_tfidf = dataReader.get_tfidf_artists()
# _ = dataReader.get_tfidf_album()
recommender_list1 = [
# Random,
# TopPop,
ItemKNNCBFRecommender,
# UserKNNCBRecommender,
ItemKNNCFRecommender,
UserKNNCFRecommender,
# P3alphaRecommender,
# RP3betaRecommender,
# MatrixFactorization_BPR_Cython,
# MatrixFactorization_FunkSVD_Cython,
SLIM_BPR_Cython,
# ItemKNNCFRecommenderFAKESLIM,
# PureSVDRecommender,
SLIMElasticNetRecommender
]
# ITEM CB, ITEM CF, USER CF, RP3BETA, PURE SVD
recommender_list2 = [
# Random,
# TopPop,
ItemKNNCBFRecommender,
# UserKNNCBRecommender,
# ItemKNNCFPageRankRecommender,
ItemKNNCFRecommender,
UserKNNCFRecommender,
# P3alphaRecommender,
# RP3betaRecommender,
# MatrixFactorization_BPR_Cython,
# MatrixFactorization_FunkSVD_Cython,
SLIM_BPR_Cython,
SLIMElasticNetRecommender
# PureSVDRecommender
]
from Base.Evaluation.Evaluator import SequentialEvaluator
evaluator = SequentialEvaluator(URM_test, URM_train, exclude_seen=True)
output_root_path = "result_experiments/"
# If directory does not exist, create
if not os.path.exists(output_root_path):
os.makedirs(output_root_path)
logFile = open(output_root_path + "result_all_algorithms.txt", "a")
try:
recommender_class = HybridRecommender
print("Algorithm: {}".format(recommender_class))
'''
Our optimal run
'''
recommender_list = recommender_list1 # + recommender_list2 # + recommender_list3
onPop = False
# On pop it used to choose if have dynamic weights for
recommender = recommender_class(URM_train, ICM, recommender_list, URM_PageRank_train=URM_PageRank_train,
dynamic=False, UCM_train=UCM_tfidf,
URM_validation=URM_validation, onPop=onPop)
lambda_i = 0.1
lambda_j = 0.05
old_similrity_matrix = None
num_factors = 395
l1_ratio = 1e-06
# Variabili secondo intervallo
alphaRP3_2 = 0.9223827655310622
betaRP3_2 = 0.2213306613226453
num_factors_2 = 391
recommender.fit(**
{
"topK": [10, 33, 160, 761, 490],
"shrink": [8, 26, 2, -1, -1],
"pop": [280],
"weights": [0.33804686720093335, 1.3092081994688194, 0.642288869881126, 0.18883962446529368,
1.9317211019160747],
"final_weights": [1, 1],
"force_compute_sim": False, # not evaluate_algorithm,
"feature_weighting_index": 0,
"epochs": 150,
'lambda_i': [0.0], 'lambda_j': [1.0153577332223556e-08], 'SLIM_lr': [0.1],
'alphaP3': [0.4121720883248633],
'alphaRP3': [0.8582865731462926],
'betaRP': [0.2814208416833668],
'l1_ratio': 3.020408163265306e-06,
'alpha': 0.0014681984611695231,
'tfidf': [True],
"weights_to_dweights": -1,
"filter_top_pop_len": 0})
print("TEST")
print("Starting Evaluations...")
# to indicate if plotting for lenght or for pop
results_run, results_run_string, target_recommendations = evaluator.evaluateRecommender(recommender,
plot_stats=False,
onPop=onPop)
print("Algorithm: {}, results: \n{}".format([rec.RECOMMENDER_NAME for rec in recommender.recommender_list],
results_run_string))
logFile.write("Algorithm: {}, results: \n{} time: {}".format(
[rec.RECOMMENDER_NAME for rec in recommender.recommender_list], results_run_string, time.time()))
logFile.flush()
if not evaluate_algorithm:
target_playlist = dataReader.get_target_playlist()
md.assign_recomendations_to_correct_playlist(target_playlist, target_recommendations)
md.make_CSV_file(target_playlist, filename)
print('File {} created!'.format(filename))
except Exception as e:
traceback.print_exc()
logFile.write("Algorithm: {} - Exception: {}\n".format(recommender_class, str(e)))
logFile.flush()
evaluator_validation, ICM_target=ICM, model_to_use="incremental")
# We compute the similarity matrix resulting from a RP3beta recommender
# Note that we have not included the code for parameter tuning, which should be done
cf_parameters = {
'topK': 500,
'alpha': 0.9,
'beta': 0.7,
'normalize_similarity': True
}
recommender_collaborative = RP3betaRecommender(URM_train)
recommender_collaborative.fit(**cf_parameters)
result_dict, result_string = evaluator_test.evaluateRecommender(
recommender_collaborative)
print("CF recommendation quality is: {}".format(result_string))
# We get the similarity matrix
# The similarity is a scipy.sparse matrix of shape |items|x|items|
similarity_collaborative = recommender_collaborative.W_sparse.copy()
# We instance and fit the feature weighting algorithm, it takes as input:
# - The train URM
# - The ICM
# - The collaborative similarity matrix
# Note that we have not included the code for parameter tuning, which should be done as those are just default parameters
fw_parameters = {
'epochs': 200,
'learning_rate': 0.0001,
def run():
evaluate_algorithm = True
delete_old_computations = False
slim_after_hybrid = False
# delete_previous_intermediate_computations()
# if not evaluate_algorithm:
# delete_previous_intermediate_computations()
# else:
# print("ATTENTION: old intermediate computations kept, pay attention if running with all_train")
# delete_previous_intermediate_computations()
filename = "hybrid_ICB_ICF_UCF_SLIM_ELASTIC_local_08052.csv"
dataReader = RS_Data_Loader(all_train=not evaluate_algorithm)
URM_train = dataReader.get_URM_train()
URM_PageRank_train = dataReader.get_page_rank_URM()
URM_validation = dataReader.get_URM_validation()
URM_test = dataReader.get_URM_test()
ICM = dataReader.get_ICM()
UCM_tfidf = dataReader.get_tfidf_artists()
# _ = dataReader.get_tfidf_album()
recommender_list1 = [
# Random,
# TopPop,
ItemKNNCBFRecommender,
# UserKNNCBRecommender,
ItemKNNCFRecommender,
UserKNNCFRecommender,
# P3alphaRecommender,
# RP3betaRecommender,
# MatrixFactorization_BPR_Cython,
# MatrixFactorization_FunkSVD_Cython,
SLIM_BPR_Cython,
# ItemKNNCFRecommenderFAKESLIM,
# PureSVDRecommender,
SLIMElasticNetRecommender
]
# ITEM CB, ITEM CF, USER CF, RP3BETA, PURE SVD
recommender_list2 = [
# Random,
# TopPop,
ItemKNNCBFRecommender,
# UserKNNCBRecommender,
ItemKNNCFRecommender,
UserKNNCFRecommender,
# P3alphaRecommender,
# RP3betaRecommender,
# MatrixFactorization_BPR_Cython,
# MatrixFactorization_FunkSVD_Cython,
SLIM_BPR_Cython,
# ItemKNNCFRecommenderFAKESLIM,
# PureSVDRecommender,
SLIMElasticNetRecommender
]
from Base.Evaluation.Evaluator import SequentialEvaluator
evaluator = SequentialEvaluator(URM_test, URM_train, exclude_seen=True)
output_root_path = "result_experiments/"
# If directory does not exist, create
if not os.path.exists(output_root_path):
os.makedirs(output_root_path)
logFile = open(output_root_path + "result_all_algorithms.txt", "a")
try:
recommender_class = HybridRecommender
print("Algorithm: {}".format(recommender_class))
'''
Our optimal run
'''
recommender_list = recommender_list1 + recommender_list2 # + recommender_list3
d_weights = [
[0.5469789514168496, 1.5598358421050373, 1.1505851198615593, 0.2540023047558251, 0.9403502151872645] + [
0] * len(recommender_list2),
[0] * len(recommender_list1) + [0.5205017325111618, 1.6831295912149837, 1.6560707664775454,
0.3144197724407203, 1.9912784665282535]
]
onPop = False
# On pop it used to choose if have dynamic weights for
recommender = recommender_class(URM_train, ICM, recommender_list, URM_PageRank_train=URM_PageRank_train,
dynamic=True, UCM_train=UCM_tfidf, d_weights=d_weights,
URM_validation=URM_validation, onPop=onPop)
recommender.fit(**
{
"topK": [10, 33, 160, 761, 490] + [10, 33, 160, 761, 490],
"shrink": [8, 26, 2, -1, -1] + [8, 26, 2, -1, -1],
"pop": [30],
"weights": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
"final_weights": [1, 1],
"force_compute_sim": False, # not evaluate_algorithm,
"feature_weighting_index": [0, 0],
"epochs": 150,
'lambda_i': [0.0, 0.0], 'lambda_j': [1.0153577332223556e-08, 1.0153577332223556e-08],
'SLIM_lr': [0.1, 0.1],
'alphaP3': [0.4121720883248633],
'alphaRP3': [0.8582865731462926],
'betaRP': [0.2814208416833668],
'l1_ratio': [3.020408163265306e-06, 3.020408163265306e-06],
'alpha': [0.0014681984611695231, 0.0014681984611695231],
'tfidf': [True, True],
"weights_to_dweights": -1,
"filter_top_pop_len": 0})
print("TEST")
print("Starting Evaluations...")
# to indicate if plotting for lenght or for pop
results_run, results_run_string, target_recommendations = evaluator.evaluateRecommender(recommender,
plot_stats=True,
onPop=onPop)
print("Algorithm: {}, results: \n{}".format([rec.RECOMMENDER_NAME for rec in recommender.recommender_list],
results_run_string))
logFile.write("Algorithm: {}, results: \n{} time: {} \n".format(
[rec.RECOMMENDER_NAME for rec in recommender.recommender_list], results_run_string, time.time()))
logFile.flush()
if not evaluate_algorithm:
target_playlist = dataReader.get_target_playlist()
md.assign_recomendations_to_correct_playlist(target_playlist, target_recommendations)
md.make_CSV_file(target_playlist, filename)
print('File {} created!'.format(filename))
except Exception as e:
traceback.print_exc()
logFile.write("Algorithm: {} - Exception: {}\n".format(recommender_class, str(e)))
logFile.flush()
