class BaseMatrixFactorizationRecommender(BaseRecommender): """ This class refers to a BaseRecommender KNN which uses matrix factorization, it provides functions to compute item's score as well as a function to save the W_matrix The prediction for cold users will always be -inf for ALL items """ def __init__(self, URM_train): super(BaseMatrixFactorizationRecommender, self).__init__(URM_train) self._cold_user_mask = np.ediff1d(self.URM_train.indptr) == 0 self._cold_user_KNN_model_available = False self._warm_user_KNN_mask = np.zeros(len(self._cold_user_mask), dtype=np.bool) if self._cold_user_mask.any(): print("{}: Detected {} ({:.2f} %) cold users.".format( self.RECOMMENDER_NAME, self._cold_user_mask.sum(), self._cold_user_mask.sum() / len(self._cold_user_mask) * 100)) def _get_cold_user_mask(self): return self._cold_user_mask def set_URM_train(self, URM_train_new, estimate_model_for_cold_users=False, topK=100, **kwargs): """ :param URM_train_new: :param estimate_item_similarity_for_cold_users: Set to TRUE if you want to estimate the item-item similarity for cold users to be used as in a KNN algorithm :param topK: 100 :param kwargs: :return: """ assert self.URM_train.shape == URM_train_new.shape, "{}: set_URM_train old and new URM train have different shapes".format( self.RECOMMENDER_NAME) if len(kwargs) > 0: print( "{}: set_URM_train keyword arguments not supported for this recommender class. Received: {}" .format(self.RECOMMENDER_NAME, kwargs)) self.URM_train = check_matrix(URM_train_new.copy(), 'csr', dtype=np.float32) self.URM_train.eliminate_zeros() if estimate_model_for_cold_users == "itemKNN": print("{}: Estimating ItemKNN model from ITEM latent factors...". format(self.RECOMMENDER_NAME)) W_sparse = compute_W_sparse_from_item_latent_factors( self.ITEM_factors, topK=topK) self._ItemKNNRecommender = ItemKNNCustomSimilarityRecommender( self.URM_train) self._ItemKNNRecommender.fit(W_sparse, topK=topK) self._cold_user_KNN_model_available = True self._warm_user_KNN_mask = np.ediff1d(self.URM_train.indptr) > 0 print( "{}: Estimating ItemKNN model from ITEM latent factors... done!" .format(self.RECOMMENDER_NAME)) elif estimate_model_for_cold_users == "mean_item_factors": print( "{}: Estimating USER latent factors from ITEM latent factors..." .format(self.RECOMMENDER_NAME)) self._cold_user_mask = np.ediff1d(self.URM_train.indptr) == 0 profile_length = np.ediff1d(self.URM_train.indptr) profile_length_sqrt = np.sqrt(profile_length) self.USER_factors = self.URM_train.dot(self.ITEM_factors) #Divide every row for the sqrt of the profile length for user_index in range(self.n_users): if profile_length_sqrt[user_index] > 0: self.USER_factors[ user_index, :] /= profile_length_sqrt[user_index] print( "{}: Estimating USER latent factors from ITEM latent factors... done!" .format(self.RECOMMENDER_NAME)) def _compute_item_score(self, user_id_array, items_to_compute=None): """ USER_factors is n_users x n_factors ITEM_factors is n_items x n_factors The prediction for cold users will always be -inf for ALL items :param user_id_array: :param items_to_compute: :return: """ assert self.USER_factors.shape[1] == self.ITEM_factors.shape[1], \ "{}: User and Item factors have inconsistent shape".format(self.RECOMMENDER_NAME) assert self.USER_factors.shape[0] > user_id_array.max(),\ "{}: Cold users not allowed. Users in trained model are {}, requested prediction for users up to {}".format( self.RECOMMENDER_NAME, self.USER_factors.shape[0], user_id_array.max()) if items_to_compute is not None: item_scores = -np.ones( (len(user_id_array), self.ITEM_factors.shape[0]), dtype=np.float32) * np.inf item_scores[:, items_to_compute] = np.dot( self.USER_factors[user_id_array], self.ITEM_factors[items_to_compute, :].T) else: item_scores = np.dot(self.USER_factors[user_id_array], self.ITEM_factors.T) cold_users_MF_mask = self._get_cold_user_mask()[user_id_array] if cold_users_MF_mask.any(): if self._cold_user_KNN_model_available: # Add KNN scores for users cold for MF but warm in KNN model cold_users_in_MF_warm_in_KNN_mask = np.logical_and( cold_users_MF_mask, self._warm_user_KNN_mask[user_id_array]) item_scores[ cold_users_in_MF_warm_in_KNN_mask, :] = self._ItemKNNRecommender._compute_item_score( user_id_array[cold_users_in_MF_warm_in_KNN_mask], items_to_compute=items_to_compute) # Set cold users as those neither in MF nor in KNN cold_users_MF_mask = np.logical_and( cold_users_MF_mask, np.logical_not(cold_users_in_MF_warm_in_KNN_mask)) # Set as -inf all remaining cold user scores item_scores[cold_users_MF_mask, :] = -np.ones_like( item_scores[cold_users_MF_mask, :]) * np.inf return item_scores def saveModel(self, folder_path, file_name=None): if file_name is None: file_name = self.RECOMMENDER_NAME print("{}: Saving model in file '{}'".format(self.RECOMMENDER_NAME, folder_path + file_name)) dictionary_to_save = { "USER_factors": self.USER_factors, "ITEM_factors": self.ITEM_factors, "_cold_user_mask": self._cold_user_mask } pickle.dump(dictionary_to_save, open(folder_path + file_name, "wb"), protocol=pickle.HIGHEST_PROTOCOL) print("{}: Saving complete".format(self.RECOMMENDER_NAME, folder_path + file_name))
class BaseMatrixFactorizationRecommender(BaseRecommender): """ This class refers to a BaseRecommender KNN which uses matrix factorization, it provides functions to compute item's score as well as a function to save the W_matrix The prediction for cold users will always be -inf for ALL items """ def __init__(self, URM_train): super(BaseMatrixFactorizationRecommender, self).__init__(URM_train) self.use_bias = False self._cold_user_KNN_model_available = False self._warm_user_KNN_mask = np.zeros(len(self._get_cold_user_mask()), dtype=np.bool) def set_URM_train(self, URM_train_new, estimate_model_for_cold_users=False, topK=100, **kwargs): """ :param URM_train_new: :param estimate_item_similarity_for_cold_users: Set to TRUE if you want to estimate the item-item similarity for cold users to be used as in a KNN algorithm :param topK: 100 :param kwargs: :return: """ assert self.URM_train.shape == URM_train_new.shape, "{}: set_URM_train old and new URM train have different shapes".format( self.RECOMMENDER_NAME) if len(kwargs) > 0: print( "{}: set_URM_train keyword arguments not supported for this recommender class. Received: {}" .format(self.RECOMMENDER_NAME, kwargs)) self.URM_train = check_matrix(URM_train_new.copy(), 'csr', dtype=np.float32) self.URM_train.eliminate_zeros() if estimate_model_for_cold_users == "itemKNN": print("{}: Estimating ItemKNN model from ITEM latent factors...". format(self.RECOMMENDER_NAME)) W_sparse = compute_W_sparse_from_item_latent_factors( self.ITEM_factors, topK=topK) self._ItemKNNRecommender = ItemKNNCustomSimilarityRecommender( self.URM_train) self._ItemKNNRecommender.fit(W_sparse, topK=topK) self._cold_user_KNN_model_available = True self._warm_user_KNN_mask = np.ediff1d(self.URM_train.indptr) > 0 print( "{}: Estimating ItemKNN model from ITEM latent factors... done!" .format(self.RECOMMENDER_NAME)) elif estimate_model_for_cold_users == "mean_item_factors": print( "{}: Estimating USER latent factors from ITEM latent factors..." .format(self.RECOMMENDER_NAME)) self._cold_user_mask = np.ediff1d(self.URM_train.indptr) == 0 profile_length = np.ediff1d(self.URM_train.indptr) profile_length_sqrt = np.sqrt(profile_length) self.USER_factors = self.URM_train.dot(self.ITEM_factors) #Divide every row for the sqrt of the profile length for user_index in range(self.n_users): if profile_length_sqrt[user_index] > 0: self.USER_factors[ user_index, :] /= profile_length_sqrt[user_index] print( "{}: Estimating USER latent factors from ITEM latent factors... done!" .format(self.RECOMMENDER_NAME)) ######################################################################################################### ########## ########## ########## COMPUTE ITEM SCORES ########## ########## ########## ######################################################################################################### def _compute_item_score(self, user_id_array, items_to_compute=None): """ USER_factors is n_users x n_factors ITEM_factors is n_items x n_factors The prediction for cold users will always be -inf for ALL items :param user_id_array: :param items_to_compute: :return: """ assert self.USER_factors.shape[1] == self.ITEM_factors.shape[1], \ "{}: User and Item factors have inconsistent shape".format(self.RECOMMENDER_NAME) assert self.USER_factors.shape[0] > user_id_array.max(),\ "{}: Cold users not allowed. Users in trained model are {}, requested prediction for users up to {}".format( self.RECOMMENDER_NAME, self.USER_factors.shape[0], user_id_array.max()) if items_to_compute is not None: item_scores = -np.ones( (len(user_id_array), self.ITEM_factors.shape[0]), dtype=np.float32) * np.inf item_scores[:, items_to_compute] = np.dot( self.USER_factors[user_id_array], self.ITEM_factors[items_to_compute, :].T) else: item_scores = np.dot(self.USER_factors[user_id_array], self.ITEM_factors.T) # No need to select only the specific negative items or warm users because the -inf score will not change if self.use_bias: item_scores += self.ITEM_bias + self.GLOBAL_bias item_scores = (item_scores.T + self.USER_bias[user_id_array]).T item_scores = self._compute_item_score_postprocess_for_cold_users( user_id_array, item_scores) item_scores = self._compute_item_score_postprocess_for_cold_items( item_scores) return item_scores def _compute_item_score_postprocess_for_cold_users(self, user_id_array, item_scores): """ Remove cold users from the computed item scores, setting them to -inf Or estimate user factors with specified method :param user_id_array: :param item_scores: :return: """ cold_users_batch_mask = self._get_cold_user_mask()[user_id_array] # Set as -inf all cold user scores if cold_users_batch_mask.any(): if self._cold_user_KNN_model_available: # Add KNN scores for users cold for MF but warm in KNN model cold_users_in_MF_warm_in_KNN_mask = np.logical_and( cold_users_batch_mask, self._warm_user_KNN_mask[user_id_array]) item_scores[ cold_users_in_MF_warm_in_KNN_mask, :] = self._ItemKNNRecommender._compute_item_score( user_id_array[cold_users_in_MF_warm_in_KNN_mask], items_to_compute=items_to_compute) # Set cold users as those neither in MF nor in KNN cold_users_batch_mask = np.logical_and( cold_users_batch_mask, np.logical_not(cold_users_in_MF_warm_in_KNN_mask)) # Set as -inf all remaining cold user scores item_scores[cold_users_batch_mask, :] = -np.ones_like( item_scores[cold_users_batch_mask, :]) * np.inf return item_scores ######################################################################################################### ########## ########## ########## LOAD AND SAVE ########## ########## ########## ######################################################################################################### def saveModel(self, folder_path, file_name=None): if file_name is None: file_name = self.RECOMMENDER_NAME print("{}: Saving model in file '{}'".format(self.RECOMMENDER_NAME, folder_path + file_name)) data_dict_to_save = { "USER_factors": self.USER_factors, "ITEM_factors": self.ITEM_factors, "use_bias": self.use_bias, "_cold_user_mask": self._cold_user_mask } if self.use_bias: data_dict_to_save["ITEM_bias"] = self.ITEM_bias data_dict_to_save["USER_bias"] = self.USER_bias data_dict_to_save["GLOBAL_bias"] = self.GLOBAL_bias dataIO = DataIO(folder_path=folder_path) dataIO.save_data(file_name=file_name, data_dict_to_save=data_dict_to_save) print("{}: Saving complete".format(self.RECOMMENDER_NAME, folder_path + file_name))