# NUOVA AGGIUNTA
reader = dataReader()
URM_train = reader.mat_complete
ICM_Art = matTrack_Artist
ICM_Alb = matTrack_Album
recommender = HybridRecommender(URM_train)
item = ItemKNNCFRecommender(URM_train)
user = UserKNNCFRecommender(URM_train)
SLIM = MultiThreadSLIM_ElasticNet(URM_train=URM_train)
item.fit(topK=800, shrink=10, similarity='cosine', normalize=True)
user.fit(topK=70, shrink=22, similarity='cosine', normalize=True)
SLIM.fit(l1_penalty=1e-05,
l2_penalty=0,
positive_only=True,
topK=150,
alpha=0.00415637376180466)
recommender.fit(ICM_Art,
ICM_Alb,
item=item,
user=user,
SLIM=SLIM,
w_itemcf=1.1,
w_usercf=0.6,
w_cbart=0.3,
w_cbalb=0.6,
w_slim=0.8,
w_svd=0.6)
file = open("SubmissionNEW2.csv", "w")
file.write("playlist_id,track_ids")
class Hybrid005AlphaRecommender(BaseRecommender):
"""Hybrid004AlphaRecommender recommender"""
RECOMMENDER_NAME = "Hybrid005AlphaRecommender"
def __init__(self, data: DataObject):
super(Hybrid005AlphaRecommender, self).__init__(data.urm_train)
self.data = data
self.cold_recommender = Hybrid100AlphaRecommender(data)
self.cold_recommender.fit()
# rec1 = RP3betaRecommender(data.urm_train)
# rec1.fit(topK=20, alpha=0.12, beta=0.24)
# rec2 = ItemKNNCFRecommender(data.urm_train)
# rec2.fit(topK=22, shrink=850, similarity='jaccard', feature_weighting='BM25')
# self.warm_2_recommender = ItemKNNSimilarityHybridRecommender(data.urm_train, rec1.W_sparse, rec2.W_sparse)
urm = data.urm_train
urm = sps.vstack([data.urm_train, data.icm_all_augmented.T])
urm = urm.tocsr()
self.warm_recommender = MultiThreadSLIM_ElasticNet(data.urm_train)
self.warm_2_3_recommender = MultiThreadSLIM_ElasticNet(data.urm_train)
self.warm_1_recommender = Hybrid101AlphaRecommender(data)
def fit(self):
# self.warm_recommender.fit(topK=30, shrink=30, feature_weighting="none", similarity="jaccard")
# self.warm_2_recommender.fit(alpha=0.9, topK=50)
self.warm_1_recommender.fit()
try:
self.warm_recommender.load_model("SLIM_ElasticNet_ICM",
"FULL_URM_topK=100_l1_ratio=0.04705_alpha=0.00115_positive_only=True_max_iter=35")
except:
self.warm_recommender.fit(topK=100, l1_ratio=0.04705, alpha=0.00115, positive_only=True, max_iter=35)
self.warm_recommender.save_model("SLIM_ElasticNet_ICM",
"FULL_URM_topK=100_l1_ratio=0.04705_alpha=0.00115_positive_only=True_max_iter=35")
def recommend(self, user_id_array, cutoff=None, remove_seen_flag=True, items_to_compute=None,
remove_top_pop_flag=False, remove_CustomItems_flag=False, return_scores=False):
if user_id_array in self.data.ids_ultra_cold_users:
return self.cold_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[0][1]:
return self.cold_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[1][1]:
return self.warm_1_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[2][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[3][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[4][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[5][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[6][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[7][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[8][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[9][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[10][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.urm_train_users_by_type[11][1]:
return self.warm_recommender.recommend(user_id_array, cutoff=cutoff)
elif user_id_array in self.data.ids_cold_user:
return self.cold_recommender.recommend(user_id_array, cutoff=cutoff)
if not os.path.exists(output_root_path):
os.makedirs(output_root_path)
logFile = open(output_root_path + "result_all_algorithms.txt", "a")
for recommender_class in recommender_list:
try:
print("Algorithm: {}".format(recommender_class))
recommender = recommender_class(URM_train)
SLIM = MultiThreadSLIM_ElasticNet(URM_train=URM_train)
SLIM.fit(l1_penalty=1e-05,
l2_penalty=0,
positive_only=True,
topK=300,
alpha=4e-5)
item = ItemKNNCFRecommender(URM_train)
W_sparse_CF = item.fit(topK=800,
shrink=8,
similarity='cosine',
normalize=True)
CBAlb = ItemKNNCBFRecommender(ICM=ICM_Alb, URM_train=URM_train)
CBAlb.fit(topK=10,
shrink=16,
similarity='cosine',
normalize=True,
feature_weighting="none")
CBArt = ItemKNNCBFRecommender(ICM=ICM_Art, URM_train=URM_train)
CBArt.fit(topK=12,
def runParameterSearch_Collaborative(recommender_class,
URM_train,
ICM_1,
ICM_2,
metric_to_optimize="PRECISION",
evaluator_validation=None,
evaluator_test=None,
evaluator_validation_earlystopping=None,
output_root_path="result_experiments/",
parallelizeKNN=True,
n_cases=100):
from ParameterTuning.AbstractClassSearch import DictionaryKeys
# If directory does not exist, create
if not os.path.exists(output_root_path):
os.makedirs(output_root_path)
try:
output_root_path_rec_name = output_root_path + recommender_class.RECOMMENDER_NAME
parameterSearch = BayesianSearch(
recommender_class,
evaluator_validation=evaluator_validation,
evaluator_test=evaluator_test)
if recommender_class in [TopPop, Random]:
recommender = recommender_class(URM_train)
recommender.fit()
output_file = open(
output_root_path_rec_name + "_BayesianSearch.txt", "a")
result_dict, result_baseline = evaluator_validation.evaluateRecommender(
recommender)
output_file.write(
"ParameterSearch: Best result evaluated on URM_validation. Results: {}"
.format(result_baseline))
pickle.dump(
result_dict.copy(),
open(output_root_path_rec_name + "_best_result_validation",
"wb"),
protocol=pickle.HIGHEST_PROTOCOL)
result_dict, result_baseline = evaluator_test.evaluateRecommender(
recommender)
output_file.write(
"ParameterSearch: Best result evaluated on URM_test. Results: {}"
.format(result_baseline))
pickle.dump(result_dict.copy(),
open(output_root_path_rec_name + "_best_result_test",
"wb"),
protocol=pickle.HIGHEST_PROTOCOL)
output_file.close()
return
##########################################################################################################
if recommender_class is UserKNNCFRecommender:
similarity_type_list = ['cosine']
run_KNNCFRecommender_on_similarity_type_partial = partial(
run_KNNCFRecommender_on_similarity_type,
parameterSearch=parameterSearch,
URM_train=URM_train,
n_cases=n_cases,
output_root_path=output_root_path_rec_name,
metric_to_optimize=metric_to_optimize)
if parallelizeKNN:
pool = PoolWithSubprocess(processes=int(2), maxtasksperchild=1)
resultList = pool.map(
run_KNNCFRecommender_on_similarity_type_partial,
similarity_type_list)
else:
for similarity_type in similarity_type_list:
run_KNNCFRecommender_on_similarity_type_partial(
similarity_type)
return
##########################################################################################################
if recommender_class is ItemKNNCFRecommender:
similarity_type_list = ['cosine']
run_KNNCFRecommender_on_similarity_type_partial = partial(
run_KNNCFRecommender_on_similarity_type,
parameterSearch=parameterSearch,
URM_train=URM_train,
n_cases=n_cases,
output_root_path=output_root_path_rec_name,
metric_to_optimize=metric_to_optimize)
if parallelizeKNN:
pool = PoolWithSubprocess(processes=int(2), maxtasksperchild=1)
resultList = pool.map(
run_KNNCFRecommender_on_similarity_type_partial,
similarity_type_list)
else:
for similarity_type in similarity_type_list:
run_KNNCFRecommender_on_similarity_type_partial(
similarity_type)
return
##########################################################################################################
# if recommender_class is MultiThreadSLIM_RMSE:
#
# hyperparamethers_range_dictionary = {}
# hyperparamethers_range_dictionary["topK"] = [50, 100]
# hyperparamethers_range_dictionary["l1_penalty"] = [1e-2, 1e-3, 1e-4]
# hyperparamethers_range_dictionary["l2_penalty"] = [1e-2, 1e-3, 1e-4]
#
#
# recommenderDictionary = {DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
# DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
# DictionaryKeys.FIT_POSITIONAL_ARGS: dict(),
# DictionaryKeys.FIT_KEYWORD_ARGS: dict(),
# DictionaryKeys.FIT_RANGE_KEYWORD_ARGS: hyperparamethers_range_dictionary}
#
#
##########################################################################################################
if recommender_class is P3alphaRecommender:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [
5, 10, 20, 50, 100, 150, 200, 300, 400, 500, 600, 700, 800
]
hyperparamethers_range_dictionary["alpha"] = range(0, 2)
hyperparamethers_range_dictionary["normalize_similarity"] = [
True, False
]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS:
dict(),
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is HybridRecommender:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["w_itemcf"] = [
x * 0.1 + 1 for x in range(0, 10)
]
hyperparamethers_range_dictionary["w_usercf"] = [
x * 0.1 for x in range(0, 10)
]
hyperparamethers_range_dictionary["w_cbart"] = [
x * 0.1 for x in range(0, 10)
]
hyperparamethers_range_dictionary["w_cbalb"] = [
x * 0.1 for x in range(0, 10)
]
hyperparamethers_range_dictionary["w_slim"] = [
x * 0.1 for x in range(0, 10)
]
#hyperparamethers_range_dictionary["w_svd"] = [x * 0.05 for x in range(0, 20)]
#hyperparamethers_range_dictionary["w_rp3"] = [x * 0.05 for x in range(0, 20)]
item = ItemKNNCFRecommender(URM_train)
user = UserKNNCFRecommender(URM_train)
SLIM = MultiThreadSLIM_ElasticNet(URM_train=URM_train)
item.fit(topK=800, shrink=10, similarity='cosine', normalize=True)
user.fit(topK=70, shrink=22, similarity='cosine', normalize=True)
SLIM.fit(l1_penalty=1e-05,
l2_penalty=0,
positive_only=True,
topK=150,
alpha=0.00415637376180466)
CBArt = ItemKNNCBFRecommender(ICM=ICM_1, URM_train=URM_train)
CBArt.fit(topK=160,
shrink=5,
similarity='cosine',
normalize=True,
feature_weighting="none")
CBAlb = ItemKNNCBFRecommender(ICM=ICM_2, URM_train=URM_train)
CBAlb.fit(topK=160,
shrink=5,
similarity='cosine',
normalize=True,
feature_weighting="none")
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS: {
"ICM_Art": ICM_1,
"ICM_Alb": ICM_2,
"item": item,
"user": user,
"SLIM": SLIM,
"CBArt": CBArt,
"CBAlb": CBAlb,
},
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is RP3betaRecommender:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [
5, 10, 20, 50, 100, 150, 200, 300, 400, 500, 600, 700, 800
]
hyperparamethers_range_dictionary["alpha"] = range(0, 2)
hyperparamethers_range_dictionary["beta"] = range(0, 2)
hyperparamethers_range_dictionary["normalize_similarity"] = [True]
hyperparamethers_range_dictionary["implicit"] = [True]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS:
dict(),
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is MatrixFactorization_FunkSVD_Cython:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["sgd_mode"] = ["adagrad", "adam"]
# hyperparamethers_range_dictionary["epochs"] = [1, 5, 10, 20, 30, 50, 70, 90, 110]
hyperparamethers_range_dictionary["num_factors"] = range(
100, 1000, 20)
hyperparamethers_range_dictionary["reg"] = [0.0, 1e-3, 1e-6, 1e-9]
hyperparamethers_range_dictionary["learning_rate"] = [
1e-2, 1e-3, 1e-4, 1e-5
]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS: {
"validation_every_n": 5,
"stop_on_validation": True,
"evaluator_object": evaluator_validation_earlystopping,
"lower_validatons_allowed": 20,
"validation_metric": metric_to_optimize
},
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is FunkSVD:
hyperparamethers_range_dictionary = {}
# hyperparamethers_range_dictionary["epochs"] = [1, 5, 10, 20, 30, 50, 70, 90, 110]
hyperparamethers_range_dictionary["num_factors"] = range(
100, 1000, 20)
hyperparamethers_range_dictionary["reg"] = [
0.0, 1e-03, 1e-06, 1e-09
]
hyperparamethers_range_dictionary["learning_rate"] = [1e-02, 1e-03]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS:
dict(),
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is MatrixFactorization_AsySVD_Cython:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["sgd_mode"] = ["adagrad", "adam"]
# hyperparamethers_range_dictionary["epochs"] = [1, 5, 10, 20, 30, 50, 70, 90, 110]
hyperparamethers_range_dictionary["num_factors"] = range(
100, 500, 10)
hyperparamethers_range_dictionary["batch_size"] = [
100, 200, 300, 400
]
hyperparamethers_range_dictionary["positive_reg"] = [
0.0, 1e-3, 1e-6, 1e-9
]
hyperparamethers_range_dictionary["negative_reg"] = [
0.0, 1e-3, 1e-6, 1e-9
]
hyperparamethers_range_dictionary["learning_rate"] = [
1e-2, 1e-3, 1e-4, 1e-5
]
hyperparamethers_range_dictionary["user_reg"] = [
1e-3, 1e-4, 1e-5, 1e-6
]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {
'positive_threshold': 1
},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS: {
"validation_every_n": 5,
"stop_on_validation": True,
"evaluator_object": evaluator_validation_earlystopping,
"lower_validatons_allowed": 20,
"validation_metric": metric_to_optimize
},
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is PureSVDRecommender:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["num_factors"] = list(
range(0, 250, 5))
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS: {},
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
#########################################################################################################
if recommender_class is SLIM_BPR_Cython:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [800, 900, 1000, 1200]
# hyperparamethers_range_dictionary["epochs"] = [1, 5, 10, 20, 30, 50, 70, 90, 110]
hyperparamethers_range_dictionary["sgd_mode"] = ["adagrad"]
hyperparamethers_range_dictionary["lambda_i"] = [1e-6]
hyperparamethers_range_dictionary["lambda_j"] = [1e-9]
hyperparamethers_range_dictionary["learning_rate"] = [
0.01, 0.001, 1e-4, 1e-5, 0.1
]
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {
'train_with_sparse_weights': True,
'symmetric': True,
'positive_threshold': 1
},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS: {
"validation_every_n": 10,
"stop_on_validation": True,
"evaluator_object": evaluator_validation_earlystopping,
"lower_validatons_allowed": 3,
"validation_metric": metric_to_optimize
},
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
##########################################################################################################
if recommender_class is MultiThreadSLIM_ElasticNet:
hyperparamethers_range_dictionary = {}
hyperparamethers_range_dictionary["topK"] = [
3300, 4300, 5300, 6300, 7300
]
hyperparamethers_range_dictionary["l1_penalty"] = [
1e-5, 1e-6, 1e-4, 1e-3
]
hyperparamethers_range_dictionary["l2_penalty"] = [1e-4]
hyperparamethers_range_dictionary["alpha"] = range(0, 1)
recommenderDictionary = {
DictionaryKeys.CONSTRUCTOR_POSITIONAL_ARGS: [URM_train],
DictionaryKeys.CONSTRUCTOR_KEYWORD_ARGS: {},
DictionaryKeys.FIT_POSITIONAL_ARGS:
dict(),
DictionaryKeys.FIT_KEYWORD_ARGS:
dict(),
DictionaryKeys.FIT_RANGE_KEYWORD_ARGS:
hyperparamethers_range_dictionary
}
#########################################################################################################
## Final step, after the hyperparameter range has been defined for each type of algorithm
best_parameters = parameterSearch.search(
recommenderDictionary,
n_cases=n_cases,
output_root_path=output_root_path_rec_name,
metric=metric_to_optimize)
except Exception as e:
print("On recommender {} Exception {}".format(recommender_class,
str(e)))
traceback.print_exc()
error_file = open(output_root_path + "ErrorLog.txt", "a")
error_file.write("On recommender {} Exception {}\n".format(
recommender_class, str(e)))
error_file.close()