def xgboost_data_prediction(self, user_id_array, relevant_items_boost,
cutoff_Boost):
dict_song_pop = ged.tracks_popularity()
# elements to add for each song
user_list = user_id_array.tolist()
tracks_duration_list = np.array(self.tracks['duration_sec']).reshape(
(-1, 1))[:, 0].tolist()
song_pop = np.array(
[[dict_song_pop[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()]).reshape(
(-1, 1))
playlist_length = np.array(
[[int(ged.lenght_playlist(self.getUserProfile(user)))] *
cutoff_Boost for user in user_list]).reshape((-1, 1))
playlist_pop = np.array([[
int(
ged.playlist_popularity(self.getUserProfile(user),
dict_song_pop))
] * cutoff_Boost for user in user_list]).reshape((-1, 1))
'''
# ucm_batch = self.UCM_train[user_list].toarray()
dim_ucm = int(len(user_list) * 20)
ucm_batch = np.array([self.UCM_dense[user] for _ in range(cutoff_Boost)
for user in user_list]).reshape(dim_ucm, -1)
dim_icm = int(len(relevant_items_boost) * 20)
icm_batch = np.array([[self.ICM_dense[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()], dtype=int).reshape(dim_icm, -1)
'''
tracks_duration = np.array(
[[tracks_duration_list[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()]).reshape(
(-1, 1))
relevant_items_boost = relevant_items_boost.reshape(-1, 1)
return np.concatenate([
relevant_items_boost, song_pop, playlist_pop, playlist_length,
tracks_duration
],
axis=1) # icm_batch , ucm_batch],
def mean_pl_pop(self, URM, songs, pop_dict):
pop_list = []
for song in songs:
pop = 0
relevant_users = np.argwhere(URM[:, song])[:, 0]
for user in relevant_users:
pop += int(
ged.playlist_popularity(self.getUserProfile(user),
pop_dict))
if relevant_users.shape[0] == 0:
pop_list.append(0)
continue
pop = int(pop / relevant_users.shape[0])
pop_list.append(pop)
return pop_list
n_users_evaluated += 1
user_id = test_user_batch_array[batch_user_index]
recommended_items = recommended_items_batch_list[
batch_user_index]
# Being the URM CSR, the indices are the non-zero column indexes
relevant_items = get_user_relevant_items(user_id, URM_test)
is_relevant = np.in1d(recommended_items,
relevant_items,
assume_unique=True)
user_profile = URM_train.indices[
URM_train.indptr[user_id]:URM_train.indptr[user_id +
1]]
key_pop = int(
ged.playlist_popularity(user_profile,
pop_dict=dict_song_pop))
key_len = int(ged.lenght_playlist(user_profile))
for cutoff in [10]:
is_relevant_current_cutoff = is_relevant[0:cutoff]
recommended_items_current_cutoff = recommended_items[
0:cutoff]
current_map = map(is_relevant_current_cutoff,
relevant_items)
MAP += current_map
if time.time(
) - start_time_print > 30 or n_users_evaluated == len(
usersToEvaluate):
print(
def recommend(self,
user_id_array,
dict_pop=None,
cutoff=None,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False):
if np.isscalar(user_id_array):
user_id_array = np.atleast_1d(user_id_array)
single_user = True
else:
single_user = False
weights = self.weights
if cutoff == None:
# noinspection PyUnresolvedReferences
cutoff = self.URM_train.shape[1] - 1
else:
cutoff
cutoff_addition = 0
cutoff_Boost = cutoff + cutoff_addition
# compute the scores using the dot product
# noinspection PyUnresolvedReferences
if self.sparse_weights:
scores = []
# noinspection PyUnresolvedReferences
for recommender in self.recommender_list:
scores_batch = recommender.compute_item_score(user_id_array)
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
if remove_seen_flag:
scores_batch[
user_index, :] = self._remove_seen_on_scores(
user_id, scores_batch[user_index, :])
if remove_top_pop_flag:
scores_batch = self._remove_TopPop_on_scores(scores_batch)
if remove_CustomItems_flag:
scores_batch = self._remove_CustomItems_on_scores(
scores_batch)
scores.append(scores_batch)
final_score = np.zeros(scores[0].shape)
for score, weight in zip(scores, weights):
final_score += (score * weight)
else:
raise NotImplementedError
ranking = []
# i take the 20 elements with highest scores
not_enough_songs = True
while not_enough_songs:
not_enough_songs = False
relevant_items_boost = (-final_score).argpartition(
cutoff_Boost, axis=1)[:, 0:cutoff_Boost]
songs_in_play = []
final_relevant_items_boost = []
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
profile_list = list(self.getUserProfile(user_id))
old_rel = list(relevant_items_boost[user_index])
not_in_playlist = [x for x in old_rel if x not in profile_list]
if len(not_in_playlist) < 10:
not_enough_songs = True
cutoff_Boost += 5
print("Cutoff boost increased to {}".format(cutoff_Boost))
to_append = old_rel + profile_list
# ho lasciato apposta doppi le canzoni che appaiono sia nelle top 50 che normalmente nella playlist,
# perchè significa che sono più importanti. Basta mettere queste due list come set per togliere il problema
final_relevant_items_boost.append(to_append)
relevant_items_boost = final_relevant_items_boost
dict_song_pop = ged.tracks_popularity()
# elements to add for each song
user_list = user_id_array.tolist()
tracks_duration_list = np.array(self.tracks['duration_sec']).reshape(
(-1, 1))[:, 0].tolist()
for user_index in range(len(user_id_array)):
user_relevant_items = relevant_items_boost[user_index]
l = len(user_relevant_items)
# similarities_values = final_score[user_index, user_relevant_items].reshape((-1, 1))
user_id = user_id_array[user_index]
# Creating numpy array for training XGBoost
song_pop = np.array(
[dict_song_pop[item] for item in user_relevant_items]).reshape(
(-1, 1))
playlist_length = np.array(
[int(ged.lenght_playlist(self.getUserProfile(user_id))) / 2] *
l).reshape((-1, 1))
playlist_pop = np.array([
int(
ged.playlist_popularity(self.getUserProfile(user_id),
dict_song_pop))
] * l).reshape((-1, 1))
icm_batch = np.array([
self.ICM_dense[item] * 10 for item in user_relevant_items
]).reshape((l, -1))
tracks_duration = np.array([
tracks_duration_list[item] for item in user_relevant_items
]).reshape((-1, 1))
relevant_items_boost_user = np.asarray(
user_relevant_items).reshape(-1, 1)
newTrainXGBoost = np.concatenate([
relevant_items_boost_user, song_pop, playlist_pop,
playlist_length, tracks_duration, icm_batch
],
axis=1)
ranking.append(
get_top_items(newTrainXGBoost, self.getUserProfile(user_id)))
try:
final_ranking = np.asarray(ranking, dtype=int)
except ValueError:
# Error here: ValueError: setting an array element with a sequence.
print(ranking)
print(user_id_array)
print(1)
a = 1
raise ValueError
assert final_ranking.shape[0] == len(
user_id_array
), "user_id_array shape:{}; final ranking shape: {} and {}".format(
len(user_id_array), final_ranking.shape, final_ranking)
assert final_ranking.shape[1] == 10
return final_ranking
def recommend(self,
user_id,
dict_pop=None,
cutoff=None,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False):
weights = self.weights
if cutoff is None:
# noinspection PyUnresolvedReferences
n = self.URM_train.shape[1] - 1
else:
n = cutoff
# compute the scores using the dot product
# noinspection PyUnresolvedReferences
if self.sparse_weights:
scores = []
user_profile = self.URM_train[user_id]
# noinspection PyUnresolvedReferences
for recommender in self.recommender_list:
scores_batch = recommender.compute_item_score(user_id)
if remove_seen_flag:
scores_batch = self._remove_seen_on_scores(
user_id, scores_batch)
if remove_top_pop_flag:
scores_batch = self._remove_TopPop_on_scores(scores_batch)
if remove_CustomItems_flag:
scores_batch = self._remove_CustomItems_on_scores(
scores_batch)
scores.append(scores_batch)
try:
assert (len(scores) == len(weights))
except:
print(
"Weights and scores from similarities have two different lenghts: {} and {}"
.format(len(scores), len(weights)))
raise TypeError
# Weight is the number of items to extract for each ranking
final_ranking = []
rankings = []
if self.dynamic:
pop = [100, 200]
user_profile_pop = self.URM_train.indices[
self.URM_train.indptr[user_id]:self.URM_train.
indptr[user_id + 1]]
level = int(ged.playlist_popularity(user_profile_pop,
dict_pop))
weights = self.change_weights(level, pop)
# needed since we have to take first the more important recommendations from more important recommender
# if we don't reach the aimed number of songs
sorted_scores = [
x for _, x in sorted(zip(weights, scores),
key=lambda pair: pair[0])
]
weights.sort(reverse=True)
scores = sorted_scores
for score, weight in zip(scores, weights):
relevant_items_partition = (-score).argpartition(n)[0:n]
relevant_items_partition_sorting = np.argsort(
-score[relevant_items_partition])
ranking = relevant_items_partition[
relevant_items_partition_sorting]
rankings.append(ranking)
final_ranking = self.add_non_present_items(
final_ranking, ranking, weight, n)
if len(final_ranking) != n:
final_ranking = self.fill_missing_items(
final_ranking, rankings, n)
else:
raise NotImplementedError
user_profile = self.URM_train.indices[
self.URM_train.indptr[user_id]:self.URM_train.indptr[user_id +
1]]
user_ratings = self.URM_train.data[
self.URM_train.indptr[user_id]:self.URM_train.indptr[user_id +
1]]
relevant_weights = self.W[user_profile]
scores = relevant_weights.T.dot(user_ratings)
return final_ranking
# If is a scalar transform it in a 1-cell array
if np.isscalar(user_id_array):
user_id_array = np.atleast_1d(user_id_array)
single_user = True
else:
single_user = False
if cutoff is None:
cutoff = self.URM_train.shape[1] - 1
# Compute the scores using the model-specific function
# Vectorize over all users in user_id_array
scores_batch = self.compute_item_score(user_id_array)
# if self.normalize:
# # normalization will keep the scores in the same range
# # of value of the ratings in dataset
# user_profile = self.URM_train[user_id]
#
# rated = user_profile.copy()
# rated.data = np.ones_like(rated.data)
# if self.sparse_weights:
# den = rated.dot(self.W_sparse).toarray().ravel()
# else:
# den = rated.dot(self.W).ravel()
# den[np.abs(den) < 1e-6] = 1.0 # to avoid NaNs
# scores /= den
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
if remove_seen_flag:
scores_batch[user_index, :] = self._remove_seen_on_scores(
user_id, scores_batch[user_index, :])
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
# relevant_items_partition = (-scores_user).argpartition(cutoff)[0:cutoff]
# relevant_items_partition_sorting = np.argsort(-scores_user[relevant_items_partition])
# ranking = relevant_items_partition[relevant_items_partition_sorting]
#
# ranking_list.append(ranking)
if remove_top_pop_flag:
scores_batch = self._remove_TopPop_on_scores(scores_batch)
if remove_CustomItems_flag:
scores_batch = self._remove_CustomItems_on_scores(scores_batch)
# scores_batch = np.arange(0,3260).reshape((1, -1))
# scores_batch = np.repeat(scores_batch, 1000, axis = 0)
# relevant_items_partition is block_size x cutoff
relevant_items_partition = (-scores_batch).argpartition(
cutoff, axis=1)[:, 0:cutoff]
# Get original value and sort it
# [:, None] adds 1 dimension to the array, from (block_size,) to (block_size,1)
# This is done to correctly get scores_batch value as [row, relevant_items_partition[row,:]]
relevant_items_partition_original_value = scores_batch[
np.arange(scores_batch.shape[0])[:,
None], relevant_items_partition]
relevant_items_partition_sorting = np.argsort(
-relevant_items_partition_original_value, axis=1)
ranking = relevant_items_partition[
np.arange(relevant_items_partition.shape[0])[:, None],
relevant_items_partition_sorting]
ranking_list = ranking.tolist()
# Return single list for one user, instead of list of lists
if single_user:
ranking_list = ranking_list[0]
return ranking_list
def recommend(self,
user_id_array,
dict_pop=None,
cutoff=None,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False):
if np.isscalar(user_id_array):
user_id_array = np.atleast_1d(user_id_array)
single_user = True
else:
single_user = False
weights = self.weights
if cutoff == None:
# noinspection PyUnresolvedReferences
cutoff = self.URM_train.shape[1] - 1
else:
cutoff
cutoff_addition = 0
cutoff_Boost = cutoff + cutoff_addition
# compute the scores using the dot product
# noinspection PyUnresolvedReferences
if self.sparse_weights:
scores = []
# noinspection PyUnresolvedReferences
for recommender in self.recommender_list:
if recommender.__class__ in [HybridRecommenderXGBoost]:
scores.append(
self.compute_score_hybrid(
recommender,
user_id_array,
dict_pop,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False))
continue
scores_batch = recommender.compute_item_score(user_id_array)
# scores_batch = np.ravel(scores_batch) # because i'm not using batch
if remove_seen_flag:
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
scores_batch[
user_index, :] = self._remove_seen_on_scores(
user_id, scores_batch[user_index, :])
if remove_top_pop_flag:
scores_batch = self._remove_TopPop_on_scores(scores_batch)
if remove_CustomItems_flag:
scores_batch = self._remove_CustomItems_on_scores(
scores_batch)
scores.append(scores_batch)
final_score = np.zeros(scores[0].shape)
for score, weight in zip(scores, weights):
final_score += (score * weight)
else:
raise NotImplementedError
# i take the 20 elements with highest scores
relevant_items_boost = (-final_score).argpartition(
cutoff_Boost, axis=1)[:, 0:cutoff_Boost]
# relevant_items_partition = (-final_score).argpartition(cutoff, axis=1)[:, 0:cutoff]
relevant_items_partition = relevant_items_boost
relevant_items_partition_original_value = final_score[
np.arange(final_score.shape[0])[:, None], relevant_items_partition]
relevant_items_partition_sorting = np.argsort(
-relevant_items_partition_original_value, axis=1)
ranking = relevant_items_partition[
np.arange(relevant_items_partition.shape[0])[:, None],
relevant_items_partition_sorting]
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
test_items = get_test_items(user_id, self.URM_validation)
predicted_songs = ranking[user_index]
if self.label_XGboost is None:
user_label = []
for song in predicted_songs:
if song in test_items:
user_label.append(1)
else:
user_label.append(0)
self.label_XGboost = np.asarray(user_label)
else:
user_label = []
for song in predicted_songs:
if song in test_items:
user_label.append(1)
else:
user_label.append(0)
self.label_XGboost = np.concatenate(
[self.label_XGboost, user_label], axis=0)
# Creating numpy array for training XGBoost
scores_list = []
for score in scores:
single_score = np.zeros(ranking.shape)
for user, ranking_line in enumerate(ranking):
single_score[user] = score[user, ranking_line]
scores_list.append(single_score.reshape(-1, 1))
# scores_list.append(ranking.reshape(-1, 1))
user_list = user_id_array.tolist()
tracks_duration_list = np.array(self.tracks['duration_sec']).reshape(
(-1, 1))[:, 0].tolist()
song_pop = np.array(
[[self.dict_song_pop[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()]).reshape(
(-1, 1))
playlist_length = np.array(
[[int(ged.lenght_playlist(get_test_items(user, self.URM_train)))] *
cutoff_Boost for user in user_list]).reshape((-1, 1))
playlist_pop = np.array([[
int(
ged.playlist_popularity(get_test_items(user, self.URM_train),
self.dict_song_pop))
] * cutoff_Boost for user in user_list]).reshape((-1, 1))
'''
# ucm_batch = self.UCM_train[user_list].toarray()
dim_ucm = int(len(user_list) * 20)
ucm_batch = np.array([self.UCM_dense[user] for _ in range(cutoff_Boost)
for user in user_list]).reshape(dim_ucm, -1)
dim_icm = int(len(relevant_items_boost) * 20)
icm_batch = np.array([[self.ICM_dense[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()]).reshape(dim_icm, -1)
'''
tracks_duration = np.array(
[[tracks_duration_list[item] for item in relevant_line]
for relevant_line in relevant_items_boost.tolist()]).reshape(
(-1, 1))
relevant_items_boost = relevant_items_boost.reshape(-1, 1)
# to_concatenate = [relevant_items_boost, song_pop, playlist_pop, playlist_length,
# tracks_duration] + scores_list
to_concatenate = scores_list
newTrainXGBoost = np.concatenate(
to_concatenate, # icm_batch, ucm_batch],
axis=1)
if self.xgb_model_ready:
print("QUI")
dPred = xgb.DMatrix(newTrainXGBoost)
preds = self.xgbModel.predict(dPred)
ranking = []
ordered_tracks = []
current_user_id = 0
for track_idx in range(newTrainXGBoost.shape[0]):
ordered_tracks.append(
(relevant_items_boost[track_idx], preds[track_idx]))
if track_idx % cutoff_Boost == 0 and track_idx != 0:
ordered_tracks.sort(key=lambda elem: elem[1], reverse=True)
ordered_tracks = [
track_id[0] for track_id in ordered_tracks
][:10]
ranking.append(np.asarray(ordered_tracks).reshape(-1))
ordered_tracks = []
current_user_id += 1
ranking = np.asarray(ranking)
elif not self.xgb_model_ready:
if self.first_time:
self.first_time = False
self.trainXGBoost = sparse.lil_matrix(newTrainXGBoost,
dtype=int)
elif not self.first_time:
self.trainXGBoost = sparse.vstack(
[self.trainXGBoost, newTrainXGBoost], dtype=int)
return ranking
def recommend(self,
user_id_array,
dict_pop=None,
cutoff=None,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False):
if np.isscalar(user_id_array):
user_id_array = np.atleast_1d(user_id_array)
single_user = True
else:
single_user = False
weights = self.weights
if cutoff == None:
# noinspection PyUnresolvedReferences
cutoff = self.URM_train.shape[1] - 1
else:
cutoff
cutoff_addition = 10
cutoff_Boost = cutoff + cutoff_addition
# compute the scores using the dot product
# noinspection PyUnresolvedReferences
if self.sparse_weights:
scores = []
# noinspection PyUnresolvedReferences
for recommender in self.recommender_list:
if recommender.__class__ in [HybridRecommenderXGBoost]:
scores.append(
self.compute_score_hybrid(
recommender,
user_id_array,
dict_pop,
remove_seen_flag=True,
remove_top_pop_flag=False,
remove_CustomItems_flag=False))
continue
scores_batch = recommender.compute_item_score(user_id_array)
# scores_batch = np.ravel(scores_batch) # because i'm not using batch
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
if remove_seen_flag:
scores_batch[
user_index, :] = self._remove_seen_on_scores(
user_id, scores_batch[user_index, :])
if remove_top_pop_flag:
scores_batch = self._remove_TopPop_on_scores(scores_batch)
if remove_CustomItems_flag:
scores_batch = self._remove_CustomItems_on_scores(
scores_batch)
scores.append(scores_batch)
final_score = np.zeros(scores[0].shape)
if self.dynamic:
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
user_profile = self.URM_train.indices[
self.URM_train.indptr[user_id]:self.URM_train.
indptr[user_id + 1]]
if self.onPop:
level = int(
ged.playlist_popularity(user_profile, dict_pop))
else:
level = int(ged.lenght_playlist(user_profile))
# weights = self.change_weights(user_id)
weights = self.change_weights(level, self.pop)
assert len(weights) == len(
scores), "Scores and weights have different lengths"
final_score_line = np.zeros(scores[0].shape[1])
if sum(weights) > 0:
for score, weight in zip(scores, weights):
final_score_line += score[user_index] * weight
final_score[user_index] = final_score_line
else:
for score, weight in zip(scores, weights):
final_score += (score * weight)
else:
raise NotImplementedError
# i take the 20 elements with highest scores
relevant_items_boost = (-final_score).argpartition(
cutoff_Boost, axis=1)[:, 0:cutoff_Boost]
# if not self.xgb_model_ready:
relevant_items_partition = (-final_score).argpartition(
cutoff, axis=1)[:, 0:cutoff]
relevant_items_partition_original_value = final_score[
np.arange(final_score.shape[0])[:, None], relevant_items_partition]
relevant_items_partition_sorting = np.argsort(
-relevant_items_partition_original_value, axis=1)
ranking = relevant_items_partition[
np.arange(relevant_items_partition.shape[0])[:, None],
relevant_items_partition_sorting]
# Creating numpy array for training XGBoost
data_reader = RS_Data_Loader()
URM_train = data_reader.get_URM_train()
pred_data_xgboost = self.xgboost_data_prediction(
user_id_array, relevant_items_boost, cutoff_Boost)
param = {
'max_depth': 3, # the maximum depth of each tree
'eta': 0.3, # the training step for each iteration
'silent': 1, # logging mode - quiet
'objective':
'multi:softprob', # error evaluation for multiclass training
'num_class': 2
} # the number of classes that exist in this datset
num_round = 20
ranking = []
for user_index in range(len(user_id_array)):
user_id = user_id_array[user_index]
# if self.user_id_XGBoost is None:
# self.user_id_XGBoost = np.array([user_id] * cutoff_Boost).reshape(-1, 1)
# else:
# self.user_id_XGBoost = np.concatenate([self.user_id_XGBoost,
# np.array([user_id] *
# cutoff_Boost).reshape(-1, 1)], axis=0)
train_xgboost = self.xgboost_data_training(user_id, URM_train)
half_play = int(train_xgboost.shape[0] / 2)
labels_train = np.array([1] * half_play + [0] * half_play)
dtrain = xgb.DMatrix(train_xgboost, label=labels_train)
bst = xgb.train(param, dtrain, num_round)
user_recommendations = pred_data_xgboost[user_index *
cutoff_Boost:(user_index +
1) *
cutoff_Boost]
dtest = xgb.DMatrix(user_recommendations)
preds = bst.predict(dtest)
predictions = self.reorder_songs(preds,
user_recommendations)[:cutoff]
ranking.append(predictions)
print(user_id, predictions)
#
# if self.xgb_model_ready:
# print("QUI")
# preds = self.xgbModel.predict_proba(newTrainXGBoost)
# # preds = self.xgbModel.predict(newTrainXGBoost)
# ranking = []
# ordered_tracks = []
# current_user_id = 0
# current_user = user_list[current_user_id]
# for track_idx in range(newTrainXGBoost.shape[0]):
# ordered_tracks.append((relevant_items_boost[track_idx], preds[track_idx][current_user]))
#
# if track_idx % cutoff_Boost and track_idx != 0:
# ordered_tracks.sort(key=lambda elem: elem[1])
# ordered_tracks = [track_id[0] for track_id in ordered_tracks]
# ranking.append(ordered_tracks)
# ordered_tracks = []
# current_user_id += 1
#
#
# elif not self.xgb_model_ready:
# if self.first_time:
# self.first_time = False
# self.trainXGBoost = sparse.lil_matrix(newTrainXGBoost, dtype=int)
# x = self.trainXGBoost
# y = self.user_id_XGBoost
# print()
#
# elif not self.first_time:
# self.trainXGBoost = sparse.vstack([self.trainXGBoost, newTrainXGBoost], dtype=int)
# x = self.trainXGBoost
# y = 0
# Return single list for one user, instead of list of lists
# if single_user:
# ranking_list = ranking_list[0]
return ranking