def fit(self,
topK=50,
shrink=100,
similarity='cosine',
normalize=True,
feature_weighting="none",
**similarity_args):
self.topK = topK
self.shrink = shrink
if feature_weighting not in self.FEATURE_WEIGHTING_VALUES:
raise ValueError(
"Value for 'feature_weighting' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.FEATURE_WEIGHTING_VALUES, feature_weighting))
if feature_weighting == "BM25":
self.URM_train = self.URM_train.astype(np.float32)
self.URM_train = okapi_BM_25(self.URM_train.T).T
self.URM_train = check_matrix(self.URM_train, 'csr')
elif feature_weighting == "TF-IDF":
self.URM_train = self.URM_train.astype(np.float32)
self.URM_train = TF_IDF(self.URM_train.T).T
self.URM_train = check_matrix(self.URM_train, 'csr')
similarity = Compute_Similarity(self.URM_train.T,
shrink=shrink,
topK=topK,
normalize=normalize,
similarity=similarity,
**similarity_args)
self.W_sparse = similarity.compute_similarity()
self.W_sparse = check_matrix(self.W_sparse, format='csr')
def fit(self,
topK=None,
l2_norm=1e3,
normalize_matrix=False,
verbose=True):
self.verbose = verbose
start_time = time.time()
self._print("Fitting model... ")
if normalize_matrix:
# Normalize rows and then columns
self.URM_train = normalize(self.URM_train, norm="l2", axis=1)
self.URM_train = normalize(self.URM_train, norm="l2", axis=0)
self.URM_train = sps.csr_matrix(self.URM_train)
# Grahm matrix is X X^t, compute dot product
similarity = Compute_Similarity(
self.URM_train,
shrink=0,
topK=self.URM_train.shape[1],
normalize=False,
similarity="cosine",
)
grahm_matrix = similarity.compute_similarity().toarray()
diag_indices = np.diag_indices(grahm_matrix.shape[0])
grahm_matrix[diag_indices] += l2_norm
P = np.linalg.inv(grahm_matrix)
B = P / (-np.diag(P))
B[diag_indices] = 0.0
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Fitting model... done in {:.2f} {}".format(
new_time_value, new_time_unit))
# Check if the matrix should be saved in a sparse or dense format
# The matrix is sparse, regardless of the presence of the topK, if nonzero cells are less than sparse_threshold_quota %
if topK is not None:
B = similarityMatrixTopK(B, k=topK, verbose=False)
if self._is_content_sparse_check(B):
self._print("Detected model matrix to be sparse, changing format.")
self.W_sparse = check_matrix(B, format="csr", dtype=np.float32)
else:
self.W_sparse = check_matrix(B, format="npy", dtype=np.float32)
self._W_sparse_format_checked = True
self._compute_item_score = self._compute_score_W_dense
def fit(self, topK=50, shrink=100, similarity='cosine', normalize=True, **similarity_args):
self.topK = topK
self.shrink = shrink
similarity = Compute_Similarity(self.URM_train.T, shrink=shrink, topK=topK, normalize=normalize, similarity = similarity, **similarity_args)
if self.sparse_weights:
self.W_sparse = similarity.compute_similarity()
else:
self.W = similarity.compute_similarity()
self.W = self.W.toarray()
def _compute_W_sparse(self, use_incremental = False):
if use_incremental:
feature_weights = self.D_incremental
else:
feature_weights = self.D_best
self.similarity = Compute_Similarity(self.ICM.T, shrink=0, topK=self.topK,
normalize=self.normalize_similarity, row_weights=feature_weights)
self.W_sparse = self.similarity.compute_similarity()
self.sparse_weights = True
def fit(self,
topK=None,
l2_norm=1e3,
normalize_matrix=False,
verbose=True):
self.verbose = verbose
start_time = time.time()
self._print("Fitting model... ")
if normalize_matrix:
# Normalize rows and then columns
self.URM_train = normalize(self.URM_train, norm='l2', axis=1)
self.URM_train = normalize(self.URM_train, norm='l2', axis=0)
self.URM_train = sps.csr_matrix(self.URM_train)
# Grahm matrix is X^t X, compute dot product
similarity = Compute_Similarity(self.URM_train,
shrink=0,
topK=self.URM_train.shape[1],
normalize=False,
similarity="cosine")
grahm_matrix = similarity.compute_similarity().toarray()
diag_indices = np.diag_indices(grahm_matrix.shape[0])
# The Compute_Similarity object ensures the diagonal of the similarity matrix is zero
# in this case we need the diagonal as well, which is just the item popularity
item_popularity = np.ediff1d(self.URM_train.tocsc().indptr)
grahm_matrix[diag_indices] = item_popularity + l2_norm
P = np.linalg.inv(grahm_matrix)
B = P / (-np.diag(P))
B[diag_indices] = 0.0
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Fitting model... done in {:.2f} {}".format(
new_time_value, new_time_unit))
if topK is None:
self.W_sparse = B
self._W_sparse_format_checked = True
self._compute_item_score = self._compute_score_W_dense
else:
self.W_sparse = similarityMatrixTopK(B, k=topK, verbose=False)
self.W_sparse = sps.csr_matrix(self.W_sparse)
def fit(self, topK=350, shrink=10, similarity='cosine', normalize=True, force_compute_sim=True, tfidf=True,
**similarity_args):
self.topK = topK
self.shrink = shrink
self.tfidf = tfidf
if not force_compute_sim:
found = True
try:
with open(os.path.join("IntermediateComputations", "ItemCF",
"tot={}_tokK={}_shrink={}_tfidf={}.pkl".format(str(len(self.URM_train.data)),
str(self.topK), str(self.shrink),
str(self.tfidf))),
'rb') as handle:
(topK_new, shrink_new, W_sparse_new) = pickle.load(handle)
except FileNotFoundError:
print("File {} not found".format(
os.path.join("IntermediateComputations", "ItemCF",
"tot={}_tokK={}_shrink={}_tfidf={}.pkl".format(str(len(self.URM_train.data)),
str(self.topK), str(self.shrink),
str(self.tfidf)))))
found = False
if found and self.topK == topK_new and self.shrink == shrink_new:
self.W_sparse = W_sparse_new
print("Saved Item CF Similarity Matrix Used!")
return
if tfidf:
sim_matrix_pre = get_tfidf(self.URM_train)
else:
sim_matrix_pre = self.URM_train
similarity = Compute_Similarity(sim_matrix_pre, shrink=shrink, topK=topK, normalize=normalize,
similarity=similarity, **similarity_args)
if self.sparse_weights:
self.W_sparse = similarity.compute_similarity()
print('Similarity item based CF computed')
with open(os.path.join("IntermediateComputations", "ItemCF",
"tot={}_tokK={}_shrink={}_tfidf={}.pkl".format(str(len(self.URM_train.data)),
str(self.topK), str(self.shrink),
str(self.tfidf))),
'wb') as handle:
pickle.dump((self.topK, self.shrink, self.W_sparse), handle, protocol=pickle.HIGHEST_PROTOCOL)
print("Item CF similarity matrix saved")
else:
self.W = similarity.compute_similarity()
self.W = self.W.toarray()
def compute_W_sparse(self, model_to_use="best"):
if model_to_use == "last":
feature_weights = self.D_incremental
elif model_to_use == "best":
feature_weights = self.D_best
else:
assert False, "{}: compute_W_sparse, 'model_to_use' parameter not recognized".format(
self.RECOMMENDER_NAME)
self.similarity = Compute_Similarity(
self.ICM.T,
shrink=0,
topK=self.topK,
normalize=self.normalize_similarity,
row_weights=feature_weights)
self.W_sparse = self.similarity.compute_similarity()
self.W_sparse = check_matrix(self.W_sparse, format='csr')
def fit(self,
topK=50,
shrink=100,
similarity='cosine',
normalize=True,
feature_weighting="none",
**similarity_args):
self.topK = topK
self.shrink = shrink
if feature_weighting not in self.FEATURE_WEIGHTING_VALUES:
raise ValueError(
"Value for 'feature_weighting' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.FEATURE_WEIGHTING_VALUES, feature_weighting))
if feature_weighting == "BM25":
self.ICM = self.ICM.astype(np.float32)
self.ICM = okapi_BM_25(self.ICM)
elif feature_weighting == "TF-IDF":
self.ICM = self.ICM.astype(np.float32)
self.ICM = TF_IDF(self.ICM)
similarity = Compute_Similarity(self.ICM.T,
shrink=shrink,
topK=topK,
normalize=normalize,
similarity=similarity,
**similarity_args)
if self.sparse_weights:
self.W_sparse = similarity.compute_similarity()
else:
self.W = similarity.compute_similarity()
self.W = self.W.toarray()
return self.W_sparse
def _generateTrainData_low_ram(self):
print(self.RECOMMENDER_NAME + ": Generating train data")
start_time_batch = time.time()
# Here is important only the structure
self.similarity = Compute_Similarity(self.ICM.T,
shrink=0,
topK=self.topK,
normalize=False)
S_matrix_contentKNN = self.similarity.compute_similarity()
S_matrix_contentKNN = check_matrix(S_matrix_contentKNN, "csr")
self._writeLog(
self.RECOMMENDER_NAME +
": Collaborative S density: {:.2E}, nonzero cells {}".format(
self.S_matrix_target.nnz / self.S_matrix_target.shape[0]**2,
self.S_matrix_target.nnz,
))
self._writeLog(self.RECOMMENDER_NAME +
": Content S density: {:.2E}, nonzero cells {}".format(
S_matrix_contentKNN.nnz /
S_matrix_contentKNN.shape[0]**2,
S_matrix_contentKNN.nnz,
))
if self.normalize_similarity:
# Compute sum of squared
sum_of_squared_features = np.array(
self.ICM.T.power(2).sum(axis=0)).ravel()
sum_of_squared_features = np.sqrt(sum_of_squared_features)
num_common_coordinates = 0
estimated_n_samples = int(S_matrix_contentKNN.nnz *
(1 + self.add_zeros_quota) * 1.2)
self.row_list = np.zeros(estimated_n_samples, dtype=np.int32)
self.col_list = np.zeros(estimated_n_samples, dtype=np.int32)
self.data_list = np.zeros(estimated_n_samples, dtype=np.float64)
num_samples = 0
for row_index in range(self.n_items):
start_pos_content = S_matrix_contentKNN.indptr[row_index]
end_pos_content = S_matrix_contentKNN.indptr[row_index + 1]
content_coordinates = S_matrix_contentKNN.indices[
start_pos_content:end_pos_content]
start_pos_target = self.S_matrix_target.indptr[row_index]
end_pos_target = self.S_matrix_target.indptr[row_index + 1]
target_coordinates = self.S_matrix_target.indices[
start_pos_target:end_pos_target]
# Chech whether the content coordinate is associated to a non zero target value
# If true, the content coordinate has a collaborative non-zero value
# if false, the content coordinate has a collaborative zero value
is_common = np.in1d(content_coordinates, target_coordinates)
num_common_in_current_row = is_common.sum()
num_common_coordinates += num_common_in_current_row
for index in range(len(is_common)):
if num_samples == estimated_n_samples:
dataBlock = 1000000
self.row_list = np.concatenate(
(self.row_list, np.zeros(dataBlock, dtype=np.int32)))
self.col_list = np.concatenate(
(self.col_list, np.zeros(dataBlock, dtype=np.int32)))
self.data_list = np.concatenate(
(self.data_list, np.zeros(dataBlock,
dtype=np.float64)))
if is_common[index]:
# If cell exists in target matrix, add its value
# Otherwise it will remain zero with a certain probability
col_index = content_coordinates[index]
self.row_list[num_samples] = row_index
self.col_list[num_samples] = col_index
new_data_value = self.S_matrix_target[row_index, col_index]
if self.normalize_similarity:
new_data_value *= (sum_of_squared_features[row_index] *
sum_of_squared_features[col_index])
self.data_list[num_samples] = new_data_value
num_samples += 1
elif np.random.rand() <= self.add_zeros_quota:
col_index = content_coordinates[index]
self.row_list[num_samples] = row_index
self.col_list[num_samples] = col_index
self.data_list[num_samples] = 0.0
num_samples += 1
if (time.time() - start_time_batch > 30
or num_samples == S_matrix_contentKNN.nnz *
(1 + self.add_zeros_quota)):
print(self.RECOMMENDER_NAME +
": Generating train data. Sample {} ( {:.2f} %) ".format(
num_samples,
num_samples / S_matrix_contentKNN.nnz *
(1 + self.add_zeros_quota) * 100,
))
sys.stdout.flush()
sys.stderr.flush()
start_time_batch = time.time()
self._writeLog(
self.RECOMMENDER_NAME +
": Content S structure has {} out of {} ( {:.2f}%) nonzero collaborative cells"
.format(
num_common_coordinates,
S_matrix_contentKNN.nnz,
num_common_coordinates / S_matrix_contentKNN.nnz * 100,
))
# Discard extra cells at the left of the array
self.row_list = self.row_list[:num_samples]
self.col_list = self.col_list[:num_samples]
self.data_list = self.data_list[:num_samples]
data_nnz = sum(np.array(self.data_list) != 0)
data_sum = sum(self.data_list)
collaborative_nnz = self.S_matrix_target.nnz
collaborative_sum = sum(self.S_matrix_target.data)
self._writeLog(
self.RECOMMENDER_NAME +
": Nonzero collaborative cell sum is: {:.2E}, average is: {:.2E}, "
"average over all collaborative data is {:.2E}".format(
data_sum, data_sum / data_nnz, collaborative_sum /
collaborative_nnz))
def fit(self,
topK=50,
shrink=100,
similarity='cosine',
normalize=True,
force_compute_sim=True,
feature_weighting="none",
feature_weighting_index=0,
**similarity_args):
self.feature_weighting_index = feature_weighting_index
feature_weighting = self.FEATURE_WEIGHTING_VALUES[
feature_weighting_index]
self.topK = topK
self.shrink = shrink
if not force_compute_sim:
found = True
try:
with open(
os.path.join(
"IntermediateComputations", "ICB",
"tot={}_topK={}_shrink={}_featureweight={}.pkl".
format(str(len(self.URM_train.data)),
str(self.topK), str(self.shrink),
str(self.feature_weighting_index))),
'rb') as handle:
(topK_new, shrink_new, W_sparse_new) = pickle.load(handle)
except FileNotFoundError:
print("File {} not found".format(
os.path.join("IntermediateComputations",
"ContentBFMatrix.pkl")))
found = False
if found and self.topK == topK_new and self.shrink == shrink_new:
self.W_sparse = W_sparse_new
print("Saved CBF Similarity Matrix Used!")
return
if feature_weighting not in self.FEATURE_WEIGHTING_VALUES:
raise ValueError(
"Value for 'feature_weighting' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.FEATURE_WEIGHTING_VALUES, feature_weighting))
if feature_weighting == "BM25":
self.ICM = self.ICM.astype(np.float32)
self.ICM = okapi_BM_25(self.ICM)
elif feature_weighting == "TF-IDF":
self.ICM = self.ICM.astype(np.float32)
self.ICM = TF_IDF(self.ICM)
similarity = Compute_Similarity(self.ICM.T,
shrink=shrink,
topK=topK,
normalize=normalize,
similarity=similarity,
**similarity_args)
if self.sparse_weights:
self.W_sparse = similarity.compute_similarity()
with open(
os.path.join(
"IntermediateComputations", "ICB",
"tot={}_topK={}_shrink={}_featureweight={}.pkl".format(
str(len(self.URM_train.data)), str(self.topK),
str(self.shrink),
str(self.feature_weighting_index))),
'wb') as handle:
pickle.dump((self.topK, self.shrink, self.W_sparse),
handle,
protocol=pickle.HIGHEST_PROTOCOL)
print("CBF similarity matrix saved")
else:
self.W = similarity.compute_similarity()
self.W = self.W.toarray()