def use_Sbest_set_W_set_predicted_URM(self):
if self.train_with_sparse_weights:
self.W_sparse = self.S_best
self.W_sparse = check_matrix(self.W_sparse, format='csr')
else:
self.W_sparse = similarityMatrixTopK(self.S_best, k=self.topK)
self.W_sparse = check_matrix(self.W_sparse, format='csr')
# Precompute URM
self.predicted_URM = self.URM.dot(self.W_sparse)
def get_S_set_W_set_predicted_URM(self):
self.S_incremental = self.cythonEpoch.get_S()
if self.train_with_sparse_weights:
self.W_sparse = self.S_incremental
self.W_sparse = check_matrix(self.W_sparse, format='csr')
else:
self.W_sparse = similarityMatrixTopK(self.S_incremental,
k=self.topK)
self.W_sparse = check_matrix(self.W_sparse, format='csr')
# Precompute URM
self.predicted_URM = self.URM.dot(self.W_sparse)
def applyPearsonCorrelation(self):
"""
Remove from every data point the average for the corresponding column
:return:
"""
self.dataMatrix = check_matrix(self.dataMatrix, 'csc')
interactionsPerCol = np.diff(self.dataMatrix.indptr)
nonzeroCols = interactionsPerCol > 0
sumPerCol = np.asarray(self.dataMatrix.sum(axis=0)).ravel()
colAverage = np.zeros_like(sumPerCol)
colAverage[nonzeroCols] = sumPerCol[nonzeroCols] / interactionsPerCol[
nonzeroCols]
# Split in blocks to avoid duplicating the whole data structure
start_col = 0
end_col = 0
blockSize = 1000
while end_col < self.n_columns:
end_col = min(self.n_columns, end_col + blockSize)
self.dataMatrix.data[self.dataMatrix.indptr[start_col]:self.dataMatrix.indptr[end_col]] -= \
np.repeat(colAverage[start_col:end_col], interactionsPerCol[start_col:end_col])
start_col += blockSize
def applyAdjustedCosine(self):
"""
Remove from every data point the average for the corresponding row
:return:
"""
self.dataMatrix = check_matrix(self.dataMatrix, 'csr')
interactionsPerRow = np.diff(self.dataMatrix.indptr)
nonzeroRows = interactionsPerRow > 0
sumPerRow = np.asarray(self.dataMatrix.sum(axis=1)).ravel()
rowAverage = np.zeros_like(sumPerRow)
rowAverage[nonzeroRows] = sumPerRow[nonzeroRows] / interactionsPerRow[
nonzeroRows]
# Split in blocks to avoid duplicating the whole data structure
start_row = 0
end_row = 0
blockSize = 1000
while end_row < self.n_rows:
end_row = min(self.n_rows, end_row + blockSize)
self.dataMatrix.data[self.dataMatrix.indptr[start_row]:self.dataMatrix.indptr[end_row]] -= \
np.repeat(rowAverage[start_row:end_row], interactionsPerRow[start_row:end_row])
start_row += blockSize
def _train(self, verbose=True):
self.A = check_matrix(self.A, 'csr', np.float64)
print("Starting training with {} learning rate, {} l1_reg, {} epochs".
format(self.learning_rate, self.l1_reg, self.epochs))
self.W, loss, samples_per_seconds = train_multiple_epochs(
self.A, self.learning_rate, self.epochs, self.l1_reg)
self.W = sp.csr_matrix(self.W)
self.predicted_URM = self.A.dot(self.W)
def _stack(self, to_stack, param, format='csr'):
"""
Stacks a new sparse matrix under the A matrix used for training
:param to_stack: sparse matrix to add
:param param: regularization
:param format: default 'csr'
"""
tmp = check_matrix(to_stack, 'csr', dtype=np.float32)
tmp = tmp.multiply(param)
self.URM = sp.vstack((self.URM, tmp), format=format, dtype=np.float32)
def fit(self):
""" Fits the ElasticNet model """
self.model = ElasticNet(alpha=self.alpha,
l1_ratio=self.l1_ratio,
positive=self.positive_only,
fit_intercept=False,
warm_start=False,
copy_X=False,
precompute=True,
selection='random',
max_iter=100,
tol=1e-4)
# the matrix that has to be learnt
self.A = check_matrix(self.URM, format='csc', dtype=np.float32)
self._train()
def apply_feature_weighting(data, feature_weighting, K=1.2, B=0.75):
if feature_weighting is None:
return data
data = data.astype(np.float32)
if feature_weighting == "BM25-Transpose":
data = _okapi_BM_25(data.T, K1=K, B=B).T
elif feature_weighting == "BM25":
data = _okapi_BM_25(data, K1=K, B=B)
elif feature_weighting == "TF-IDF-Transpose":
data = _TF_IDF(data.T).T
elif feature_weighting == "TF-IDF":
data = _TF_IDF(data)
else:
raise NotImplementedError()
data = check_matrix(data, 'csr')
return data
def compute_similarity(self, start_col=None, end_col=None, block_size=100):
"""
Compute the similarity for the given dataset
:param self:
:param start_col: column to begin with
:param end_col: column to stop before, end_col is excluded
:return:
"""
values = []
rows = []
cols = []
start_time = time.time()
start_time_print_batch = start_time
processedItems = 0
if self.adjusted_cosine:
self.applyAdjustedCosine()
elif self.pearson_correlation:
self.applyPearsonCorrelation()
elif self.tanimoto_coefficient or self.dice_coefficient or self.tversky_coefficient:
self.useOnlyBooleanInteractions()
# We explore the matrix column-wise
self.dataMatrix = check_matrix(self.dataMatrix, 'csc')
# Compute sum of squared values to be used in normalization
sumOfSquared = np.array(self.dataMatrix.power(2).sum(axis=0)).ravel()
# Tanimoto does not require the square root to be applied
if not (self.tanimoto_coefficient or self.dice_coefficient
or self.tversky_coefficient):
sumOfSquared = np.sqrt(sumOfSquared)
if self.asymmetric_cosine:
sumOfSquared_to_1_minus_alpha = np.power(
sumOfSquared, 2 * (1 - self.asymmetric_alpha))
sumOfSquared_to_alpha = np.power(sumOfSquared,
2 * self.asymmetric_alpha)
self.dataMatrix = check_matrix(self.dataMatrix, 'csc')
start_col_local = 0
end_col_local = self.n_columns
if start_col is not None and start_col > 0 and start_col < self.n_columns:
start_col_local = start_col
if end_col is not None and end_col > start_col_local and end_col < self.n_columns:
end_col_local = end_col
start_col_block = start_col_local
this_block_size = 0
# Compute all similarities for each item using vectorization
while start_col_block < end_col_local:
end_col_block = min(start_col_block + block_size, end_col_local)
this_block_size = end_col_block - start_col_block
# All data points for a given item
item_data = self.dataMatrix[:, start_col_block:end_col_block]
item_data = item_data.toarray().squeeze()
# If only 1 feature avoid last dimension to disappear
if item_data.ndim == 1:
item_data = np.atleast_2d(item_data)
if self.use_row_weights:
this_block_weights = self.dataMatrix_weighted.T.dot(item_data)
else:
# Compute item similarities
this_block_weights = self.dataMatrix.T.dot(item_data)
for col_index_in_block in range(this_block_size):
if this_block_size == 1:
this_column_weights = this_block_weights
else:
this_column_weights = this_block_weights[:,
col_index_in_block]
columnIndex = col_index_in_block + start_col_block
this_column_weights[columnIndex] = 0.0
# Apply normalization and shrinkage, ensure denominator != 0
if self.normalize:
if self.asymmetric_cosine:
denominator = sumOfSquared_to_alpha[
columnIndex] * sumOfSquared_to_1_minus_alpha + self.shrink + 1e-6
else:
denominator = sumOfSquared[
columnIndex] * sumOfSquared + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights,
1 / denominator)
# Apply the specific denominator for Tanimoto
elif self.tanimoto_coefficient:
denominator = sumOfSquared[
columnIndex] + sumOfSquared - this_column_weights + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights,
1 / denominator)
elif self.dice_coefficient:
denominator = sumOfSquared[
columnIndex] + sumOfSquared + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights,
1 / denominator)
elif self.tversky_coefficient:
denominator = this_column_weights + \
(sumOfSquared[columnIndex] - this_column_weights)*self.tversky_alpha + \
(sumOfSquared - this_column_weights)*self.tversky_beta + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights,
1 / denominator)
# If no normalization or tanimoto is selected, apply only shrink
elif self.shrink != 0:
this_column_weights = this_column_weights / self.shrink
#this_column_weights = this_column_weights.toarray().ravel()
# Sort indices and select TopK
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
relevant_items_partition = (
-this_column_weights).argpartition(self.TopK -
1)[0:self.TopK]
relevant_items_partition_sorting = np.argsort(
-this_column_weights[relevant_items_partition])
top_k_idx = relevant_items_partition[
relevant_items_partition_sorting]
# Incrementally build sparse matrix, do not add zeros
notZerosMask = this_column_weights[top_k_idx] != 0.0
numNotZeros = np.sum(notZerosMask)
values.extend(this_column_weights[top_k_idx][notZerosMask])
rows.extend(top_k_idx[notZerosMask])
cols.extend(np.ones(numNotZeros) * columnIndex)
# Add previous block size
processedItems += this_block_size
if time.time(
) - start_time_print_batch >= 30 or end_col_block == end_col_local:
columnPerSec = processedItems / (time.time() - start_time +
1e-9)
print(
"Similarity column {} ( {:2.0f} % ), {:.2f} column/sec, elapsed time {:.2f} min"
.format(
processedItems, processedItems /
(end_col_local - start_col_local) * 100, columnPerSec,
(time.time() - start_time) / 60))
sys.stdout.flush()
sys.stderr.flush()
start_time_print_batch = time.time()
start_col_block += block_size
# End while on columns
W_sparse = sps.csr_matrix((values, (rows, cols)),
shape=(self.n_columns, self.n_columns),
dtype=np.float32)
return W_sparse
def fit(self):
self.URM = apply_feature_weighting(self.URM, self.feature_weighting, K=self.K, B=self.B)
#
# if X.dtype != np.float32:
# print("P3ALPHA fit: For memory usage reasons, we suggest to use np.float32 as dtype for the dataset")
if self.min_rating > 0:
self.URM.data[self.URM.data < self.min_rating] = 0
self.URM.eliminate_zeros()
if self.implicit:
self.URM.data = np.ones(self.URM.data.size, dtype=np.float32)
#Pui is the row-normalized urm
Pui = normalize(self.URM, norm='l1', axis=1)
#Piu is the column-normalized, "boolean" urm transposed
X_bool = self.URM.transpose(copy=True)
X_bool.data = np.ones(X_bool.data.size, np.float32)
#ATTENTION: axis is still 1 because i transposed before the normalization
Piu = normalize(X_bool, norm='l1', axis=1)
del(X_bool)
# Alfa power
if self.alpha != 1.:
Pui = Pui.power(self.alpha)
Piu = Piu.power(self.alpha)
# Final matrix is computed as Pui * Piu * Pui
# Multiplication unpacked for memory usage reasons
block_dim = 200
d_t = Piu
# Use array as it reduces memory requirements compared to lists
dataBlock = 10000000
rows = np.zeros(dataBlock, dtype=np.int32)
cols = np.zeros(dataBlock, dtype=np.int32)
values = np.zeros(dataBlock, dtype=np.float32)
numCells = 0
start_time = time.time()
start_time_printBatch = start_time
for current_block_start_row in range(0, Pui.shape[1], block_dim):
if current_block_start_row + block_dim > Pui.shape[1]:
block_dim = Pui.shape[1] - current_block_start_row
similarity_block = d_t[current_block_start_row:current_block_start_row + block_dim, :] * Pui
similarity_block = similarity_block.toarray()
for row_in_block in range(block_dim):
row_data = similarity_block[row_in_block, :]
row_data[current_block_start_row + row_in_block] = 0
best = row_data.argsort()[::-1][:self.topK]
notZerosMask = row_data[best] != 0.0
values_to_add = row_data[best][notZerosMask]
cols_to_add = best[notZerosMask]
for index in range(len(values_to_add)):
if numCells == len(rows):
rows = np.concatenate((rows, np.zeros(dataBlock, dtype=np.int32)))
cols = np.concatenate((cols, np.zeros(dataBlock, dtype=np.int32)))
values = np.concatenate((values, np.zeros(dataBlock, dtype=np.float32)))
rows[numCells] = current_block_start_row + row_in_block
cols[numCells] = cols_to_add[index]
values[numCells] = values_to_add[index]
numCells += 1
if time.time() - start_time_printBatch > 60:
print("Processed {} ( {:.2f}% ) in {:.2f} minutes. Rows per second: {:.0f}".format(
current_block_start_row,
100.0 * float(current_block_start_row) / Pui.shape[1],
(time.time() - start_time) / 60,
float(current_block_start_row) / (time.time() - start_time)))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
self.W_sparse = sp.csr_matrix((values[:numCells], (rows[:numCells], cols[:numCells])), shape=(Pui.shape[1], Pui.shape[1]))
if self.normalize_similarity:
self.W_sparse = normalize(self.W_sparse, norm='l1', axis=1)
if self.topK != False:
self.W_sparse = similarityMatrixTopK(self.W_sparse, k=self.topK)
self.W_sparse = check_matrix(self.W_sparse, format='csr')
# Precompute URM
self.predicted_URM = self.URM.dot(self.W_sparse)