def __init__(self, max_total_time_seconds, current_total_time):
max_total_time_seconds_value, max_total_time_seconds_unit = seconds_to_biggest_unit(
max_total_time_seconds)
current_total_time_seconds_value, current_total_time_seconds_unit = seconds_to_biggest_unit(
current_total_time)
message = "Total training and evaluation time is {:.2f} {}, exceeding the maximum threshold of {:.2f} {}".format(
current_total_time_seconds_value, current_total_time_seconds_unit,
max_total_time_seconds_value, max_total_time_seconds_unit)
super().__init__(message)
def compute_eigenvalues(self, lamda = None, U = None):
start_time = time.time()
if lamda is None or U is None:
print("SpectralCF: Computing adjacient_matrix...")
self.A = self._adjacient_matrix(self_connection=True)
print("SpectralCF: Computing degree_matrix...")
self.D = self._degree_matrix()
print("SpectralCF: Computing laplacian_matrix...")
self.L = self._laplacian_matrix(normalized=True)
print("SpectralCF: Computing eigenvalues...")
self.lamda, self.U = np.linalg.eig(self.L)
self.lamda = np.diag(self.lamda)
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() - start_time)
print("SpectralCF: Initialization complete in {:.2f} {}".format(new_time_value, new_time_unit))
else:
self.lamda = lamda
self.U = U
def fit(self,
topK=None,
l2_norm=1e3,
normalize_matrix=False,
verbose=True):
self.verbose = verbose
start_time = time.time()
self._print("Fitting model... ")
if normalize_matrix:
# Normalize rows and then columns
self.URM_train = normalize(self.URM_train, norm='l2', axis=1)
self.URM_train = normalize(self.URM_train, norm='l2', axis=0)
self.URM_train = sps.csr_matrix(self.URM_train)
# Grahm matrix is X^t X, compute dot product
similarity = Compute_Similarity(self.URM_train,
shrink=0,
topK=self.URM_train.shape[1],
normalize=False,
similarity="cosine")
grahm_matrix = similarity.compute_similarity().toarray()
diag_indices = np.diag_indices(grahm_matrix.shape[0])
# The Compute_Similarity object ensures the diagonal of the similarity matrix is zero
# in this case we need the diagonal as well, which is just the item popularity
item_popularity = np.ediff1d(self.URM_train.tocsc().indptr)
grahm_matrix[diag_indices] = item_popularity + l2_norm
P = np.linalg.inv(grahm_matrix)
B = P / (-np.diag(P))
B[diag_indices] = 0.0
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Fitting model... done in {:.2f} {}".format(
new_time_value, new_time_unit))
# Check if the matrix should be saved in a sparse or dense format
# The matrix is sparse, regardless of the presence of the topK, if nonzero cells are less than sparse_threshold_quota %
if topK is not None:
B = similarityMatrixTopK(B, k=topK, verbose=False)
if self._is_content_sparse_check(B):
self._print("Detected model matrix to be sparse, changing format.")
self.W_sparse = check_matrix(B, format='csr', dtype=np.float32)
else:
self.W_sparse = check_matrix(B, format='npy', dtype=np.float32)
self._W_sparse_format_checked = True
self._compute_item_score = self._compute_score_W_dense
def _convert_sec_list_into_biggest_unit(data_list):
mean_sec, stddev_sec = _mean_and_stdd_of_list(data_list)
_, new_time_unit, data_list = seconds_to_biggest_unit(mean_sec,
data_array=data_list)
mean_new_unit, stddev_new_unit = _mean_and_stdd_of_list(data_list)
return mean_sec, stddev_sec, new_time_unit, mean_new_unit, stddev_new_unit
def fit(self,
topK=None,
l2_norm=1e3,
normalize_matrix=False,
verbose=True):
self.verbose = verbose
start_time = time.time()
self._print("Fitting model... ")
if normalize_matrix:
# Normalize rows and then columns
self.URM_train = normalize(self.URM_train, norm='l2', axis=1)
self.URM_train = normalize(self.URM_train, norm='l2', axis=0)
self.URM_train = sps.csr_matrix(self.URM_train)
# Grahm matrix is X^t X, compute dot product
similarity = Compute_Similarity(self.URM_train,
shrink=0,
topK=self.URM_train.shape[1],
normalize=False,
similarity="cosine")
grahm_matrix = similarity.compute_similarity().toarray()
diag_indices = np.diag_indices(grahm_matrix.shape[0])
# The Compute_Similarity object ensures the diagonal of the similarity matrix is zero
# in this case we need the diagonal as well, which is just the item popularity
item_popularity = np.ediff1d(self.URM_train.tocsc().indptr)
grahm_matrix[diag_indices] = item_popularity + l2_norm
P = np.linalg.inv(grahm_matrix)
B = P / (-np.diag(P))
B[diag_indices] = 0.0
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Fitting model... done in {:.2f} {}".format(
new_time_value, new_time_unit))
if topK is None:
self.W_sparse = B
self._W_sparse_format_checked = True
self._compute_item_score = self._compute_score_W_dense
else:
self.W_sparse = similarityMatrixTopK(B, k=topK, verbose=False)
self.W_sparse = sps.csr_matrix(self.W_sparse)
def fit(self,
num_factors=100,
l1_ratio=0.5,
solver="multiplicative_update",
init_type="random",
beta_loss="frobenius",
verbose=False,
random_seed=None):
assert l1_ratio >= 0 and l1_ratio <= 1, "{}: l1_ratio must be between 0 and 1, provided value was {}".format(
self.RECOMMENDER_NAME, l1_ratio)
if solver not in self.SOLVER_VALUES:
raise ValueError(
"Value for 'solver' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.SOLVER_VALUES.keys(), solver))
if init_type not in self.INIT_VALUES:
raise ValueError(
"Value for 'init_type' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.INIT_VALUES, init_type))
if beta_loss not in self.BETA_LOSS_VALUES:
raise ValueError(
"Value for 'beta_loss' not recognized. Acceptable values are {}, provided was '{}'"
.format(self.BETA_LOSS_VALUES, beta_loss))
start_time = time.time()
self._print("Computing NMF decomposition...")
nmf_solver = NMF(n_components=num_factors,
init=init_type,
solver=self.SOLVER_VALUES[solver],
beta_loss=beta_loss,
random_state=random_seed,
l1_ratio=l1_ratio,
shuffle=True,
verbose=verbose,
max_iter=500)
nmf_solver.fit(self.URM_train)
self.ITEM_factors = nmf_solver.components_.copy().T
self.USER_factors = nmf_solver.transform(self.URM_train)
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Computing NMF decomposition... done in {:.2f} {}".format(
new_time_value, new_time_unit))
def _convert_sec_list_into_biggest_unit(data_list):
"""
Converts a list containing seconds into an equivalent list with a bigger time unit
adjusting standard deviation as well
:param data_list:
:return:
"""
data_array = np.array(data_list)
mean_sec, stddev_sec = _mean_and_stdd_of_array(data_array)
_, new_time_unit, data_array = seconds_to_biggest_unit(
mean_sec, data_array=data_array)
mean_new_unit, stddev_new_unit = _mean_and_stdd_of_array(data_array)
return mean_sec, stddev_sec, new_time_unit, mean_new_unit, stddev_new_unit
def fit(self, num_factors=100, random_seed=None):
start_time = time.time()
self._print("Computing SVD decomposition...")
U, Sigma, QT = randomized_svd(
self.URM_train,
n_components=num_factors,
#n_iter=5,
random_state=random_seed)
U_s = U * sps.diags(Sigma)
self.USER_factors = U_s
self.ITEM_factors = QT.T
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() -
start_time)
self._print("Computing SVD decomposition... done in {:.2f} {}".format(
new_time_value, new_time_unit))
def _train_with_early_stopping(
self,
epochs_max,
epochs_min=0,
validation_every_n=None,
stop_on_validation=False,
validation_metric=None,
lower_validations_allowed=None,
evaluator_object=None,
algorithm_name="Incremental_Training_Early_Stopping"):
"""
:param epochs_max: max number of epochs the training will last
:param epochs_min: min number of epochs the training will last
:param validation_every_n: number of epochs after which the model will be evaluated and a best_model selected
:param stop_on_validation: [True/False] whether to stop the training before the max number of epochs
:param validation_metric: which metric to use when selecting the best model, higher values are better
:param lower_validations_allowed: number of contiguous validation steps required for the tranining to early-stop
:param evaluator_object: evaluator instance used to compute the validation metrics.
If multiple cutoffs are available, the first one is used
:param algorithm_name: name of the algorithm to be displayed in the output updates
:return: -
Supported uses:
- Train for max number of epochs with no validation nor early stopping:
_train_with_early_stopping(epochs_max = 100,
evaluator_object = None
epochs_min, not used
validation_every_n, not used
stop_on_validation, not used
validation_metric, not used
lower_validations_allowed, not used
)
- Train for max number of epochs with validation but NOT early stopping:
_train_with_early_stopping(epochs_max = 100,
evaluator_object = evaluator
stop_on_validation = False
validation_every_n = int value
validation_metric = metric name string
epochs_min, not used
lower_validations_allowed, not used
)
- Train for max number of epochs with validation AND early stopping:
_train_with_early_stopping(epochs_max = 100,
epochs_min = int value
evaluator_object = evaluator
stop_on_validation = True
validation_every_n = int value
validation_metric = metric name string
lower_validations_allowed = int value
)
"""
assert epochs_max > 0, "{}: Number of epochs_max must be > 0, passed was {}".format(
algorithm_name, epochs_max)
assert epochs_min >= 0, "{}: Number of epochs_min must be >= 0, passed was {}".format(
algorithm_name, epochs_min)
assert epochs_min <= epochs_max, "{}: epochs_min must be <= epochs_max, passed are epochs_min {}, epochs_max {}".format(
algorithm_name, epochs_min, epochs_max)
# Train for max number of epochs with no validation nor early stopping
# OR Train for max number of epochs with validation but NOT early stopping
# OR Train for max number of epochs with validation AND early stopping
assert evaluator_object is None or \
(evaluator_object is not None and not stop_on_validation and validation_every_n is not None and validation_metric is not None) or \
(evaluator_object is not None and stop_on_validation and validation_every_n is not None and validation_metric is not None and lower_validations_allowed is not None), \
"{}: Inconsistent parameters passed, please check the supported uses".format(algorithm_name)
start_time = time.time()
self.best_validation_metric = None
lower_validatons_count = 0
convergence = False
self.epochs_best = 0
epochs_current = 0
while epochs_current < epochs_max and not convergence:
self._run_epoch(epochs_current)
# If no validation required, always keep the latest
if evaluator_object is None:
self.epochs_best = epochs_current
# Determine whether a validaton step is required
elif (epochs_current + 1) % validation_every_n == 0:
print("{}: Validation begins...".format(algorithm_name))
self._prepare_model_for_validation()
# If the evaluator validation has multiple cutoffs, choose the first one
results_run, results_run_string = evaluator_object.evaluateRecommender(
self)
results_run = results_run[list(results_run.keys())[0]]
print("{}: {}".format(algorithm_name, results_run_string))
# Update optimal model
current_metric_value = results_run[validation_metric]
if self.best_validation_metric is None or self.best_validation_metric < current_metric_value:
print("{}: New best model found! Updating.".format(
algorithm_name))
self.best_validation_metric = current_metric_value
self._update_best_model()
self.epochs_best = epochs_current + 1
lower_validatons_count = 0
else:
lower_validatons_count += 1
if stop_on_validation and lower_validatons_count >= lower_validations_allowed and epochs_current >= epochs_min:
convergence = True
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
print(
"{}: Convergence reached! Terminating at epoch {}. Best value for '{}' at epoch {} is {:.4f}. Elapsed time {:.2f} {}"
.format(algorithm_name, epochs_current + 1,
validation_metric, self.epochs_best,
self.best_validation_metric, new_time_value,
new_time_unit))
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
print("{}: Epoch {} of {}. Elapsed time {:.2f} {}".format(
algorithm_name, epochs_current + 1, epochs_max, new_time_value,
new_time_unit))
epochs_current += 1
sys.stdout.flush()
sys.stderr.flush()
# If no validation required, keep the latest
if evaluator_object is None:
self._prepare_model_for_validation()
self._update_best_model()
# Stop when max epochs reached and not early-stopping
if not convergence:
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
if evaluator_object is not None:
print(
"{}: Terminating at epoch {}. Best value for '{}' at epoch {} is {:.4f}. Elapsed time {:.2f} {}"
.format(algorithm_name, epochs_current, validation_metric,
self.epochs_best, self.best_validation_metric,
new_time_value, new_time_unit))
else:
print("{}: Terminating at epoch {}. Elapsed time {:.2f} {}".
format(algorithm_name, epochs_current, new_time_value,
new_time_unit))
def fit(self, alpha=1., beta=0.6, min_rating=0, topK=100, implicit=False, normalize_similarity=True):
self.alpha = alpha
self.beta = beta
self.min_rating = min_rating
self.topK = topK
self.implicit = implicit
self.normalize_similarity = normalize_similarity
# if X.dtype != np.float32:
# print("RP3beta fit: For memory usage reasons, we suggest to use np.float32 as dtype for the dataset")
if self.min_rating > 0:
self.URM_train.data[self.URM_train.data < self.min_rating] = 0
self.URM_train.eliminate_zeros()
if self.implicit:
self.URM_train.data = np.ones(self.URM_train.data.size, dtype=np.float32)
#Pui is the row-normalized urm
Pui = normalize(self.URM_train, norm='l1', axis=1)
#Piu is the column-normalized, "boolean" urm transposed
X_bool = self.URM_train.transpose(copy=True)
X_bool.data = np.ones(X_bool.data.size, np.float32)
# Taking the degree of each item to penalize top popular
# Some rows might be zero, make sure their degree remains zero
X_bool_sum = np.array(X_bool.sum(axis=1)).ravel()
degree = np.zeros(self.URM_train.shape[1])
nonZeroMask = X_bool_sum!=0.0
degree[nonZeroMask] = np.power(X_bool_sum[nonZeroMask], -self.beta)
#ATTENTION: axis is still 1 because i transposed before the normalization
Piu = normalize(X_bool, norm='l1', axis=1)
del(X_bool)
# Alfa power
if self.alpha != 1.:
Pui = Pui.power(self.alpha)
Piu = Piu.power(self.alpha)
# Final matrix is computed as Pui * Piu * Pui
# Multiplication unpacked for memory usage reasons
block_dim = 200
d_t = Piu
# Use array as it reduces memory requirements compared to lists
dataBlock = 10000000
rows = np.zeros(dataBlock, dtype=np.int32)
cols = np.zeros(dataBlock, dtype=np.int32)
values = np.zeros(dataBlock, dtype=np.float32)
numCells = 0
start_time = time.time()
start_time_printBatch = start_time
for current_block_start_row in range(0, Pui.shape[1], block_dim):
if current_block_start_row + block_dim > Pui.shape[1]:
block_dim = Pui.shape[1] - current_block_start_row
similarity_block = d_t[current_block_start_row:current_block_start_row + block_dim, :] * Pui
similarity_block = similarity_block.toarray()
for row_in_block in range(block_dim):
row_data = np.multiply(similarity_block[row_in_block, :], degree)
row_data[current_block_start_row + row_in_block] = 0
best = row_data.argsort()[::-1][:self.topK]
notZerosMask = row_data[best] != 0.0
values_to_add = row_data[best][notZerosMask]
cols_to_add = best[notZerosMask]
for index in range(len(values_to_add)):
if numCells == len(rows):
rows = np.concatenate((rows, np.zeros(dataBlock, dtype=np.int32)))
cols = np.concatenate((cols, np.zeros(dataBlock, dtype=np.int32)))
values = np.concatenate((values, np.zeros(dataBlock, dtype=np.float32)))
rows[numCells] = current_block_start_row + row_in_block
cols[numCells] = cols_to_add[index]
values[numCells] = values_to_add[index]
numCells += 1
if time.time() - start_time_printBatch > 300:
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() - start_time)
self._print("Similarity column {} ({:4.1f}%), {:.2f} column/sec. Elapsed time {:.2f} {}".format(
current_block_start_row + block_dim,
100.0 * float( current_block_start_row + block_dim) / Pui.shape[1],
float( current_block_start_row + block_dim) / (time.time() - start_time),
new_time_value, new_time_unit))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
self.W_sparse = sps.csr_matrix((values[:numCells], (rows[:numCells], cols[:numCells])), shape=(Pui.shape[1], Pui.shape[1]))
if self.normalize_similarity:
self.W_sparse = normalize(self.W_sparse, norm='l1', axis=1)
if self.topK != False:
self.W_sparse = similarityMatrixTopK(self.W_sparse, k=self.topK)
self.W_sparse = check_matrix(self.W_sparse, format='csr')
def fit(self, l1_ratio=0.1, alpha=1.0, positive_only=True, topK=100):
assert l1_ratio >= 0 and l1_ratio <= 1, "{}: l1_ratio must be between 0 and 1, provided value was {}".format(
self.RECOMMENDER_NAME, l1_ratio)
self.l1_ratio = l1_ratio
self.positive_only = positive_only
self.topK = topK
# initialize the ElasticNet model
self.model = ElasticNet(alpha=alpha,
l1_ratio=self.l1_ratio,
positive=self.positive_only,
fit_intercept=False,
copy_X=False,
precompute=True,
selection='random',
max_iter=100,
tol=1e-4)
URM_train = check_matrix(self.URM_train, 'csc', dtype=np.float32)
n_items = URM_train.shape[1]
# Use array as it reduces memory requirements compared to lists
dataBlock = 10000000
rows = np.zeros(dataBlock, dtype=np.int32)
cols = np.zeros(dataBlock, dtype=np.int32)
values = np.zeros(dataBlock, dtype=np.float32)
numCells = 0
start_time = time.time()
start_time_printBatch = start_time
# fit each item's factors sequentially (not in parallel)
for currentItem in range(n_items):
# get the target column
y = URM_train[:, currentItem].toarray()
# set the j-th column of X to zero
start_pos = URM_train.indptr[currentItem]
end_pos = URM_train.indptr[currentItem + 1]
current_item_data_backup = URM_train.data[start_pos:end_pos].copy()
URM_train.data[start_pos:end_pos] = 0.0
# fit one ElasticNet model per column
self.model.fit(URM_train, y)
# self.model.coef_ contains the coefficient of the ElasticNet model
# let's keep only the non-zero values
# Select topK values
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
nonzero_model_coef_index = self.model.sparse_coef_.indices
nonzero_model_coef_value = self.model.sparse_coef_.data
local_topK = min(len(nonzero_model_coef_value) - 1, self.topK)
relevant_items_partition = (
-nonzero_model_coef_value
).argpartition(local_topK)[0:local_topK]
relevant_items_partition_sorting = np.argsort(
-nonzero_model_coef_value[relevant_items_partition])
ranking = relevant_items_partition[
relevant_items_partition_sorting]
for index in range(len(ranking)):
if numCells == len(rows):
rows = np.concatenate(
(rows, np.zeros(dataBlock, dtype=np.int32)))
cols = np.concatenate(
(cols, np.zeros(dataBlock, dtype=np.int32)))
values = np.concatenate(
(values, np.zeros(dataBlock, dtype=np.float32)))
rows[numCells] = nonzero_model_coef_index[ranking[index]]
cols[numCells] = currentItem
values[numCells] = nonzero_model_coef_value[ranking[index]]
numCells += 1
# finally, replace the original values of the j-th column
URM_train.data[start_pos:end_pos] = current_item_data_backup
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
if time.time(
) - start_time_printBatch > 300 or currentItem == n_items - 1:
self._print(
"Processed {} ({:4.1f}%) in {:.2f} {}. Items per second: {:.2f}"
.format(currentItem + 1,
100.0 * float(currentItem + 1) / n_items,
new_time_value, new_time_unit,
float(currentItem) / elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
# generate the sparse weight matrix
self.W_sparse = sps.csr_matrix(
(values[:numCells], (rows[:numCells], cols[:numCells])),
shape=(n_items, n_items),
dtype=np.float32)
def print_time_statistics_latex_table(result_folder_path,
dataset_name,
results_file_prefix_name,
other_algorithm_list=list(),
n_validation_users=None,
n_test_users=None,
KNN_similarity_to_report_list=list(
["cosine"]),
ICM_names_to_report_list=list(),
n_decimals=4):
import pickle
import numpy as np
results_file_root_name = "{}_{}".format(results_file_prefix_name,
dataset_name)
results_file = open(
result_folder_path + "..//" + results_file_root_name +
"_latex_time.txt", "w")
def mean_and_stdd_of_list(data_list):
data_list = np.array(data_list)
mean = np.mean(data_list)
if len(data_list) == 1:
stddev = 0.0
else:
stddev = np.std(data_list, ddof=1)
return mean, stddev
# Write columns
columns_datasets_list = "\t&"
columns_metrics_list = "\t\t&"
column_name_list = [
"\\begin{tabular}{@{}c@{}}Train time\\end{tabular}",
#"validation_time",
#"\\begin{tabular}{@{}c@{}}validation \\ usr/sec\end{tabular}",
"\\begin{tabular}{@{}c@{}}Recommendation\\end{tabular}",
"\\begin{tabular}{@{}c@{}}Recommendation \\\\ {[usr/s]}\\end{tabular}"
]
for dataset_name in [dataset_name]:
colum_width = len(column_name_list)
columns_datasets_list += " \\multicolumn{{{}}}{{c}}{{{}}} \t".format(
colum_width, dataset_name)
if dataset_name != [dataset_name][-1]:
columns_datasets_list += "&"
for column_name in column_name_list:
columns_metrics_list += " {} \t".format(column_name)
if column_name != column_name_list[-1]:
columns_metrics_list += "\n\t&"
columns_datasets_list += "\\\\ \n"
columns_metrics_list += "\\\\ \n"
results_file.write(columns_datasets_list)
results_file.write(columns_metrics_list)
results_file.flush()
algorithm_data_to_print_list = get_algorithm_data_to_print_list(
KNN_similarity_to_report_list=KNN_similarity_to_report_list,
ICM_names_to_report_list=ICM_names_to_report_list,
other_algorithm_list=other_algorithm_list)
for row_index in range(len(algorithm_data_to_print_list)):
algorithm = algorithm_data_to_print_list[row_index]["algorithm"]
algorithm_row_label = algorithm_data_to_print_list[row_index][
"algorithm_row_label"]
algorithm_file_name = algorithm_data_to_print_list[row_index][
"algorithm_file_name"]
result_row_string = algorithm_row_label + "\t&"
for experiment_subfolder in [result_folder_path]:
try:
result_dict = pickle.load(
open(
experiment_subfolder + algorithm_file_name +
"_metadata", "rb"))
except:
result_dict = None
if result_dict is not None:
data_list = result_dict["train_time_list"]
data_list = np.array(data_list)
data_list_not_none_mask = np.array(
[val is not None for val in data_list])
data_list = data_list[data_list_not_none_mask]
mean, stddev = mean_and_stdd_of_list(data_list)
if len(result_dict["train_time_list"]) > 1:
result_row_string += "{:.{n_decimals}f} $\\pm$ {:.{n_decimals}f} [s]\t&".format(
mean, stddev, n_decimals=n_decimals)
else:
new_time_value, new_time_unit = seconds_to_biggest_unit(
mean)
if new_time_unit == "s":
result_row_string += "{:.{n_decimals}f} [{}] \t&".format(
mean, "s", n_decimals=n_decimals)
else:
result_row_string += "{:.{n_decimals}f} [{}] / {:.{n_decimals}f} [{}]\t&".format(
mean,
"s",
new_time_value,
new_time_unit,
n_decimals=n_decimals)
# mean, stddev = mean_and_stdd_of_list(result_dict["evaluation_time_list"])
# result_row_string+= "{:.{n_decimals}f} $\pm$ {:.{n_decimals}f}\t&".format(mean, stddev, n_decimals=n_decimals)
#
# if n_validation_users is not None:
# evaluation_time = result_dict["evaluation_time_list"][-1]
# result_row_string+= "{:.0f}\t&".format(n_validation_users/evaluation_time)
# else:
# result_row_string+=" - \t&"
non_nan_test_time = []
for value in result_dict["evaluation_test_time_list"]:
if value is not None:
non_nan_test_time.append(value)
mean, stddev = mean_and_stdd_of_list(non_nan_test_time)
if len(non_nan_test_time) > 1:
result_row_string += "{:.{n_decimals}f} $\\pm$ {:.{n_decimals}f} [s]\t&".format(
mean, stddev, n_decimals=n_decimals)
else:
new_time_value, new_time_unit = seconds_to_biggest_unit(
mean)
# result_row_string+= "{:.{n_decimals}f} \t&".format(mean, n_decimals=n_decimals)
if new_time_unit == "s":
result_row_string += "{:.{n_decimals}f} [{}] \t&".format(
mean, "s", n_decimals=n_decimals)
else:
result_row_string += "{:.{n_decimals}f} [{}] / {:.{n_decimals}f} [{}]\t&".format(
mean,
"s",
new_time_value,
new_time_unit,
n_decimals=n_decimals)
if n_test_users is not None:
evaluation_time = non_nan_test_time[-1]
result_row_string += "{:.0f}\t".format(n_test_users /
evaluation_time)
else:
result_row_string += " - \t"
else:
result_row_string += " - \t& - \t& - "
result_row_string += "\\\\ \n"
results_file.write(result_row_string)
results_file.flush()
results_file.close()
def evaluateRecommender(self, recommender_object):
"""
:param recommender_object: the trained recommender object, a BaseRecommender subclass
:param URM_test_list: list of URMs to test the recommender against, or a single URM object
:param cutoff_list: list of cutoffs to be use to report the scores, or a single cutoff
"""
results_dict = {}
for cutoff in self.cutoff_list:
results_dict[cutoff] = create_empty_metrics_dict(
self.n_items, self.n_users, recommender_object.URM_train,
self.ignore_items_ID, self.ignore_users_ID, cutoff,
self.diversity_object)
start_time = time.time()
start_time_print = time.time()
n_users_evaluated = 0
if self.ignore_items_flag:
recommender_object.set_items_to_ignore(self.ignore_items_ID)
for test_user in self.usersToEvaluate:
# Being the URM CSR, the indices are the non-zero column indexes
relevant_items = self.get_user_relevant_items(test_user)
relevant_items_rating = self.get_user_test_ratings(test_user)
n_users_evaluated += 1
items_to_compute = self._get_user_specific_items_to_compute(
test_user)
recommended_items, all_items_predicted_ratings = recommender_object.recommend(
np.atleast_1d(test_user),
remove_seen_flag=self.exclude_seen,
cutoff=self.max_cutoff,
remove_top_pop_flag=False,
items_to_compute=items_to_compute,
remove_CustomItems_flag=self.ignore_items_flag,
return_scores=True)
assert len(
recommended_items
) == 1, "{}: recommended_items contained recommendations for {} users, expected was {}".format(
self.EVALUATOR_NAME, len(recommended_items), 1)
assert all_items_predicted_ratings.shape[
0] == 1, "{}: all_items_predicted_ratings contained scores for {} users, expected was {}".format(
self.EVALUATOR_NAME, all_items_predicted_ratings.shape[0],
1)
assert all_items_predicted_ratings.shape[
1] == self.n_items, "{}: all_items_predicted_ratings contained scores for {} items, expected was {}".format(
self.EVALUATOR_NAME, all_items_predicted_ratings.shape[1],
self.n_items)
recommended_items = np.array(recommended_items[0])
user_rmse = rmse(all_items_predicted_ratings[0], relevant_items,
relevant_items_rating)
recommender_object.reset_items_to_ignore()
is_relevant = np.in1d(recommended_items,
relevant_items,
assume_unique=True)
# (CNR) -------------------------------------------------
weighted_hits = np.zeros(len(recommended_items))
log_weighted_hits = np.zeros(len(recommended_items))
pos_weighted_hits = np.zeros(len(recommended_items))
pos_log_weighted_hits = np.zeros(len(recommended_items))
'''
alpha, beta, scale, pi = 100, 0.03, 1 / 15, np.pi
percentile = get_percentile(a, 45)
f = 1 / (beta * np.sqrt(2 * pi))
y_a = np.tanh(alpha * a) + scale * f * np.exp(-1 / (2 * (beta ** 2)) * (a - percentile) ** 2)
y_a = y_a / max(y_a)
'''
# es. recommended_items = [2, 7, 10, 70, 5464]
# es. is_relevant = [0, 0, 1, 0, 0]
for i in range(len(recommended_items)):
if is_relevant[i]:
weighted_hits[i] = 1 / (
1 + Settings.popularity[recommended_items[i]])
# es. weighted_hits = [1, 7, 10, 70, 5464]
pos_weighted_hits[i] = 1 / (
1 + i + Settings.popularity[recommended_items[i]])
log_weighted_hits[i] = 1 / (1 + math.log(
1 + Settings.popularity[recommended_items[i]]))
pos_log_weighted_hits[i] = 1 / (1 + i + math.log(
1 + Settings.popularity[recommended_items[i]]))
# -------------------------------------------------------
number_of_guessed_items = 0
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
# Questo array e' fondamentale.
# Dato un utente, results_current_cutoff e' nella forma [0, 1, 0, 0, 0]
# La sua lunghezza e' pari al cutoff.
# Nell'esempio riportato, il vettore ci dice che l'oggetto
# raccomandato in seconda posizione e' un hit, cioe' e' presente nel test set.
# Attenzione pero'. Gli oggetti presi in esame sono quelli del test set piu' quelli negativi
# (che sappiamo non piacere all'utente).
is_relevant_current_cutoff = is_relevant[0:cutoff]
# (CNR) -------------------------------------------------------
custom_hits = np.zeros(len(recommended_items))
for i in range(len(recommended_items)):
if is_relevant[i]:
# print('Luciano > Computing custom weight. Parameters (pop, pos, cutoff):', Settings.popularity[recommended_items[i]], i, cutoff)
custom_hits[i] = y_custom(
Settings.popularity[recommended_items[i]], i,
cutoff)
if custom_hits[i] > 1:
print(
'=============================================================='
)
print(
'Luciano > WARNING! custom_hits[{}]={}'.format(
i, custom_hits[i]))
print(
'=============================================================='
)
weighted_hits_current_cutoff = weighted_hits[0:cutoff]
log_weighted_hits_current_cutoff = log_weighted_hits[0:cutoff]
pos_weighted_hits_current_cutoff = pos_weighted_hits[0:cutoff]
pos_log_weighted_hits_current_cutoff = pos_log_weighted_hits[
0:cutoff]
custom_hits_current_cutoff = custom_hits[0:cutoff]
# -------------------------------------------------------------
recommended_items_current_cutoff = recommended_items[0:cutoff]
results_current_cutoff[
EvaluatorMetrics.ROC_AUC.value] += roc_auc(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.PRECISION.value] += precision(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.PRECISION_RECALL_MIN_DEN.
value] += precision_recall_min_denominator(
is_relevant_current_cutoff, len(relevant_items))
results_current_cutoff[
EvaluatorMetrics.RECALL.value] += recall(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NDCG.value] += ndcg(
recommended_items_current_cutoff,
relevant_items,
relevance=self.get_user_test_ratings(test_user),
at=cutoff)
number_of_guessed_items = is_relevant_current_cutoff.sum()
results_current_cutoff[
EvaluatorMetrics.HIT_RATE.
value] += is_relevant_current_cutoff.sum()
# (CNR) -------------------------------------------------
verbose = False
if verbose and is_relevant_current_cutoff.sum() > 1:
print(
'============================================================================='
)
print('user:'******'is_relevant_current_cutoff:',
is_relevant_current_cutoff)
print('recommended_items_current_cutoff:',
recommended_items_current_cutoff)
print('Warning! is_relevant_current_cutoff.sum()>1:',
is_relevant_current_cutoff.sum())
print('relevant_items:', relevant_items)
print('relevant_items_rating:', relevant_items_rating)
print('items_to_compute:', items_to_compute)
print(
'============================================================================='
)
results_current_cutoff[
EvaluatorMetrics.WEIGHTED_HIT_RATE.
value] += weighted_hits_current_cutoff.sum()
results_current_cutoff[
EvaluatorMetrics.LOG_WEIGHTED_HIT_RATE.
value] += log_weighted_hits_current_cutoff.sum()
results_current_cutoff[
EvaluatorMetrics.POS_WEIGHTED_HIT_RATE.
value] += pos_weighted_hits_current_cutoff.sum()
results_current_cutoff[
EvaluatorMetrics.POS_LOG_WEIGHTED_HIT_RATE.
value] += pos_log_weighted_hits_current_cutoff.sum()
results_current_cutoff[
EvaluatorMetrics.CUSTOM_HIT_RATE.
value] += custom_hits_current_cutoff.sum()
# -------------------------------------------------------
results_current_cutoff[EvaluatorMetrics.ARHR.value] += arhr(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.RMSE.value] += user_rmse
results_current_cutoff[
EvaluatorMetrics.MRR.value].add_recommendations(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.MAP.value].add_recommendations(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[
EvaluatorMetrics.NOVELTY.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.AVERAGE_POPULARITY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_GINI.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.SHANNON_ENTROPY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_ITEM.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_USER.value].add_recommendations(
recommended_items_current_cutoff, test_user)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_MEAN_INTER_LIST.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_HERFINDAHL.
value].add_recommendations(
recommended_items_current_cutoff)
if EvaluatorMetrics.DIVERSITY_SIMILARITY.value in results_current_cutoff:
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_SIMILARITY.
value].add_recommendations(
recommended_items_current_cutoff)
verbose = False
if verbose and test_user % 1000 == 0:
if number_of_guessed_items > 0:
print(
'Test ======================================================='
)
print('user:'******'relevant items:', relevant_items)
print('relevant items rating:', relevant_items_rating)
print('items_to_compute:\n', items_to_compute)
print('len(items_to_compute):', len(items_to_compute))
print('recommended_items:', recommended_items)
print('is_relevant:', is_relevant)
print('number_of_guessed_items:', number_of_guessed_items)
print(
'============================================================'
)
else:
print('.')
if time.time() - start_time_print > 30 or n_users_evaluated == len(
self.usersToEvaluate):
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
print(
"{}: Processed {} ( {:.2f}% ) in {:.2f} {}. Users per second: {:.0f}"
.format(
self.EVALUATOR_NAME, n_users_evaluated, 100.0 *
float(n_users_evaluated) / len(self.usersToEvaluate),
new_time_value, new_time_unit,
float(n_users_evaluated) / elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_print = time.time()
if (n_users_evaluated > 0):
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
for key in results_current_cutoff.keys():
value = results_current_cutoff[key]
if isinstance(value, Metrics_Object):
results_current_cutoff[key] = value.get_metric_value()
else:
results_current_cutoff[key] = value / n_users_evaluated
precision_ = results_current_cutoff[
EvaluatorMetrics.PRECISION.value]
recall_ = results_current_cutoff[EvaluatorMetrics.RECALL.value]
if precision_ + recall_ != 0:
# F1 micro averaged: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.104.8244&rep=rep1&type=pdf
results_current_cutoff[EvaluatorMetrics.F1.value] = 2 * (
precision_ * recall_) / (precision_ + recall_)
else:
print(
"WARNING: No users had a sufficient number of relevant items")
if self.ignore_items_flag:
recommender_object.reset_items_to_ignore()
results_run_string = get_result_string(results_dict)
return (results_dict, results_run_string)
def _compute_metrics_on_recommendation_list(self, test_user_batch_array,
recommended_items_batch_list,
scores_batch, results_dict):
assert len(recommended_items_batch_list) == len(
test_user_batch_array
), "{}: recommended_items_batch_list contained recommendations for {} users, expected was {}".format(
self.EVALUATOR_NAME, len(recommended_items_batch_list),
len(test_user_batch_array))
assert scores_batch.shape[0] == len(
test_user_batch_array
), "{}: scores_batch contained scores for {} users, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[0],
len(test_user_batch_array))
assert scores_batch.shape[
1] == self.n_items, "{}: scores_batch contained scores for {} items, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[1], self.n_items)
# Compute recommendation quality for each user in batch
for batch_user_index in range(len(recommended_items_batch_list)):
test_user = test_user_batch_array[batch_user_index]
#print(f"Evaluating user: {test_user}")
relevant_items = self.get_user_relevant_items(test_user)
# Add the RMSE to the global object, no need to loop through the various cutoffs
# This repository is not designed to ensure proper RMSE optimization
# relevant_items_rating = self.get_user_test_ratings(test_user)
#
# all_items_predicted_ratings = scores_batch[batch_user_index]
# global_RMSE_object = results_dict[self.cutoff_list[0]][EvaluatorMetrics.RMSE.value]
# global_RMSE_object.add_recommendations(all_items_predicted_ratings, relevant_items, relevant_items_rating)
# Being the URM CSR, the indices are the non-zero column indexes
recommended_items = recommended_items_batch_list[batch_user_index]
is_relevant = np.in1d(recommended_items,
relevant_items,
assume_unique=True)
#print(f"Is Relevant: {is_relevant}")
self._n_users_evaluated += 1
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
is_relevant_current_cutoff = is_relevant[0:cutoff]
recommended_items_current_cutoff = recommended_items[0:cutoff]
results_current_cutoff[
EvaluatorMetrics.ROC_AUC.value] += roc_auc(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.PRECISION.value] += precision(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.PRECISION_RECALL_MIN_DEN.
value] += precision_recall_min_denominator(
is_relevant_current_cutoff, len(relevant_items))
results_current_cutoff[
EvaluatorMetrics.RECALL.value] += recall(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NDCG.value] += ndcg(
recommended_items_current_cutoff,
relevant_items,
relevance=self.get_user_test_ratings(test_user),
at=cutoff)
results_current_cutoff[
EvaluatorMetrics.HIT_RATE.
value] += is_relevant_current_cutoff.sum()
results_current_cutoff[EvaluatorMetrics.ARHR.value] += arhr(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.MRR.value].add_recommendations(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.MAP.value].add_recommendations(
is_relevant_current_cutoff, relevant_items)
#print(results_current_cutoff[EvaluatorMetrics.MAP.value])
#print("----------------------------------------------------------------------------------------------------")
results_current_cutoff[
EvaluatorMetrics.NOVELTY.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.AVERAGE_POPULARITY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_GINI.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.SHANNON_ENTROPY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_ITEM.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_ITEM_CORRECT.
value].add_recommendations(
recommended_items_current_cutoff,
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_USER.value].add_recommendations(
recommended_items_current_cutoff, test_user)
results_current_cutoff[EvaluatorMetrics.COVERAGE_USER_CORRECT.
value].add_recommendations(
is_relevant_current_cutoff,
test_user)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_MEAN_INTER_LIST.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_HERFINDAHL.
value].add_recommendations(
recommended_items_current_cutoff)
if EvaluatorMetrics.DIVERSITY_SIMILARITY.value in results_current_cutoff:
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_SIMILARITY.
value].add_recommendations(
recommended_items_current_cutoff)
if time.time(
) - self._start_time_print > 30 or self._n_users_evaluated == len(
self.users_to_evaluate):
elapsed_time = time.time() - self._start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
self._print(
"Processed {} ( {:.2f}% ) in {:.2f} {}. Users per second: {:.0f}"
.format(
self._n_users_evaluated,
100.0 * float(self._n_users_evaluated) /
len(self.users_to_evaluate), new_time_value, new_time_unit,
float(self._n_users_evaluated) / elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
self._start_time_print = time.time()
return results_dict
def compute_similarity(self, start_col=None, end_col=None, block_size=100):
"""
Compute the similarity for the given dataset
:param self:
:param start_col: column to begin with
:param end_col: column to stop before, end_col is excluded
:return:
"""
values = []
rows = []
cols = []
start_time = time.time()
start_time_print_batch = start_time
processed_items = 0
start_col_local = 0
end_col_local = self.n_columns
if start_col is not None and start_col > 0 and start_col < self.n_columns:
start_col_local = start_col
if end_col is not None and end_col > start_col_local and end_col < self.n_columns:
end_col_local = end_col
# Compute sum of squared values
item_distance_initial = np.array(
self.dataMatrix.power(2).sum(axis=0)).ravel()
sumOfSquared = np.sqrt(item_distance_initial)
start_col_block = start_col_local
this_block_size = 0
# Compute all similarities for each item using vectorization
while start_col_block < end_col_local:
# Compute block first and last column
end_col_block = min(start_col_block + block_size, end_col_local)
this_block_size = end_col_block - start_col_block
# All data points for a given item
item_data = self.dataMatrix[:, start_col_block:end_col_block]
item_data = item_data.toarray()
# Compute item similarities
if self.use_row_weights:
this_block_weights = self.dataMatrix_weighted.T.dot(item_data)
else:
this_block_weights = self.dataMatrix.T.dot(item_data)
for col_index_in_block in range(this_block_size):
if this_block_size == 1:
this_column_weights = this_block_weights.ravel()
else:
this_column_weights = this_block_weights[:,
col_index_in_block]
columnIndex = col_index_in_block + start_col_block
# (a-b)^2 = a^2 + b^2 - 2ab
item_distance = item_distance_initial.copy()
item_distance += item_distance_initial[columnIndex]
item_distance -= 2 * this_column_weights
item_distance[columnIndex] = 0.0
if self.use_row_weights:
item_distance = np.multiply(item_distance,
self.row_weights)
if self.normalize:
denominator = sumOfSquared[columnIndex] * sumOfSquared
item_distance[denominator != 0.0] /= denominator[
denominator != 0.0]
if self.normalize_avg_row:
item_distance /= self.n_rows
nonzero_distance_mask = item_distance > 0.0
item_distance[nonzero_distance_mask] = np.sqrt(
item_distance[nonzero_distance_mask])
if self.similarity_is_exp:
item_similarity = 1 / (np.exp(item_distance) +
self.shrink + 1e-9)
elif self.similarity_is_lin:
item_similarity = 1 / (item_distance + self.shrink + 1e-9)
elif self.similarity_is_log:
item_similarity = 1 / (np.log(item_distance + 1) +
self.shrink + 1e-9)
else:
assert False
item_similarity[columnIndex] = 0.0
this_column_weights = item_similarity
# Sort indices and select TopK
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
relevant_items_partition = (
-this_column_weights).argpartition(self.TopK -
1)[0:self.TopK]
relevant_items_partition_sorting = np.argsort(
-this_column_weights[relevant_items_partition])
top_k_idx = relevant_items_partition[
relevant_items_partition_sorting]
# Incrementally build sparse matrix, do not add zeros
notZerosMask = this_column_weights[top_k_idx] != 0.0
numNotZeros = np.sum(notZerosMask)
values.extend(this_column_weights[top_k_idx][notZerosMask])
rows.extend(top_k_idx[notZerosMask])
cols.extend(np.ones(numNotZeros) * columnIndex)
# Add previous block size
start_col_block += this_block_size
processed_items += this_block_size
if time.time(
) - start_time_print_batch >= 300 or end_col_block == end_col_local:
column_per_sec = processed_items / (time.time() - start_time +
1e-9)
new_time_value, new_time_unit = seconds_to_biggest_unit(
time.time() - start_time)
print(
"Similarity column {} ({:4.1f}%), {:.2f} column/sec. Elapsed time {:.2f} {}"
.format(
processed_items, processed_items /
(end_col_local - start_col_local) * 100,
column_per_sec, new_time_value, new_time_unit))
sys.stdout.flush()
sys.stderr.flush()
start_time_print_batch = time.time()
# End while on columns
W_sparse = sps.csr_matrix((values, (rows, cols)),
shape=(self.n_columns, self.n_columns),
dtype=np.float32)
return W_sparse
def _compute_metrics_on_recommendation_list(self, test_user_batch_array,
recommended_items_batch_list,
scores_batch, results_dict):
assert len(recommended_items_batch_list) == len(
test_user_batch_array
), "{}: recommended_items_batch_list contained recommendations for {} users, expected was {}".format(
self.EVALUATOR_NAME, len(recommended_items_batch_list),
len(test_user_batch_array))
assert scores_batch.shape[0] == len(
test_user_batch_array
), "{}: scores_batch contained scores for {} users, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[0],
len(test_user_batch_array))
assert scores_batch.shape[
1] == self.n_items, "{}: scores_batch contained scores for {} items, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[1], self.n_items)
# Compute recommendation quality for each user in batch
for batch_user_index in range(len(recommended_items_batch_list)):
test_user = test_user_batch_array[batch_user_index]
relevant_items = self.get_user_relevant_items(test_user)
# Being the URM CSR, the indices are the non-zero column indexes
recommended_items = recommended_items_batch_list[batch_user_index]
is_relevant = np.in1d(recommended_items,
relevant_items,
assume_unique=True)
self._n_users_evaluated += 1
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
is_relevant_current_cutoff = is_relevant[0:cutoff]
recommended_items_current_cutoff = recommended_items[0:cutoff]
results_current_cutoff[
EvaluatorMetrics.PRECISION.value] += precision(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.PRECISION_RECALL_MIN_DEN.
value] += precision_recall_min_denominator(
is_relevant_current_cutoff, len(relevant_items))
results_current_cutoff[
EvaluatorMetrics.RECALL.value] += recall(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NDCG.value] += ndcg(
recommended_items_current_cutoff,
relevant_items,
relevance=self.get_user_test_ratings(test_user),
at=cutoff)
results_current_cutoff[
EvaluatorMetrics.ARHR.value] += arhr_all_hits(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.MRR.value].add_recommendations(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.MAP.value].add_recommendations(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[
EvaluatorMetrics.MAP_MIN_DEN.value].add_recommendations(
is_relevant_current_cutoff, relevant_items)
results_current_cutoff[
EvaluatorMetrics.HIT_RATE.value].add_recommendations(
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.NOVELTY.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.AVERAGE_POPULARITY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_GINI.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.SHANNON_ENTROPY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_ITEM.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_ITEM_HIT.
value].add_recommendations(
recommended_items_current_cutoff,
is_relevant_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.COVERAGE_USER.value].add_recommendations(
recommended_items_current_cutoff, test_user)
results_current_cutoff[EvaluatorMetrics.COVERAGE_USER_HIT.
value].add_recommendations(
is_relevant_current_cutoff,
test_user)
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_MEAN_INTER_LIST.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_HERFINDAHL.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.RATIO_SHANNON_ENTROPY.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.RATIO_DIVERSITY_HERFINDAHL.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.RATIO_DIVERSITY_GINI.
value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.RATIO_NOVELTY.value].add_recommendations(
recommended_items_current_cutoff)
results_current_cutoff[
EvaluatorMetrics.RATIO_AVERAGE_POPULARITY.
value].add_recommendations(
recommended_items_current_cutoff)
if EvaluatorMetrics.DIVERSITY_SIMILARITY.value in results_current_cutoff:
results_current_cutoff[
EvaluatorMetrics.DIVERSITY_SIMILARITY.
value].add_recommendations(
recommended_items_current_cutoff)
if time.time(
) - self._start_time_print > 300 or self._n_users_evaluated == len(
self.users_to_evaluate):
elapsed_time = time.time() - self._start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
self._print(
"Processed {} ({:4.1f}%) in {:.2f} {}. Users per second: {:.0f}"
.format(
self._n_users_evaluated,
100.0 * float(self._n_users_evaluated) /
len(self.users_to_evaluate), new_time_value, new_time_unit,
float(self._n_users_evaluated) /
elapsed_time if elapsed_time > 0.0 else np.nan))
sys.stdout.flush()
sys.stderr.flush()
self._start_time_print = time.time()
return results_dict
def _run_evaluation_on_selected_users(self, recommender_object, usersToEvaluate, block_size = None):
if block_size is None:
block_size = min(1000, int(1e8/self.n_items))
start_time = time.time()
start_time_print = time.time()
results_dict = {}
for cutoff in self.cutoff_list:
results_dict[cutoff] = create_empty_metrics_dict(self.n_items, self.n_users,
recommender_object.get_URM_train(),
self.ignore_items_ID,
self.ignore_users_ID,
cutoff,
self.diversity_object)
n_users_evaluated = 0
# Start from -block_size to ensure it to be 0 at the first block
user_batch_start = 0
user_batch_end = 0
while user_batch_start < len(self.usersToEvaluate):
user_batch_end = user_batch_start + block_size
user_batch_end = min(user_batch_end, len(usersToEvaluate))
test_user_batch_array = np.array(usersToEvaluate[user_batch_start:user_batch_end])
user_batch_start = user_batch_end
# Compute predictions for a batch of users using vectorization, much more efficient than computing it one at a time
recommended_items_batch_list, scores_batch = recommender_object.recommend(test_user_batch_array,
remove_seen_flag=self.exclude_seen,
cutoff = self.max_cutoff,
remove_top_pop_flag=False,
remove_CustomItems_flag=self.ignore_items_flag,
return_scores = True
)
assert len(recommended_items_batch_list) == len(test_user_batch_array), "{}: recommended_items_batch_list contained recommendations for {} users, expected was {}".format(
self.EVALUATOR_NAME, len(recommended_items_batch_list), len(test_user_batch_array))
assert scores_batch.shape[0] == len(test_user_batch_array), "{}: scores_batch contained scores for {} users, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[0], len(test_user_batch_array))
assert scores_batch.shape[1] == self.n_items, "{}: scores_batch contained scores for {} items, expected was {}".format(
self.EVALUATOR_NAME, scores_batch.shape[1], self.n_items)
# Compute recommendation quality for each user in batch
for batch_user_index in range(len(recommended_items_batch_list)):
test_user = test_user_batch_array[batch_user_index]
relevant_items = self.get_user_relevant_items(test_user)
relevant_items_rating = self.get_user_test_ratings(test_user)
all_items_predicted_ratings = scores_batch[batch_user_index]
user_rmse = rmse(all_items_predicted_ratings, relevant_items, relevant_items_rating)
# Being the URM CSR, the indices are the non-zero column indexes
recommended_items = recommended_items_batch_list[batch_user_index]
is_relevant = np.in1d(recommended_items, relevant_items, assume_unique=True)
n_users_evaluated += 1
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
is_relevant_current_cutoff = is_relevant[0:cutoff]
recommended_items_current_cutoff = recommended_items[0:cutoff]
results_current_cutoff[EvaluatorMetrics.ROC_AUC.value] += roc_auc(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.PRECISION.value] += precision(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.PRECISION_RECALL_MIN_DEN.value] += precision_recall_min_denominator(is_relevant_current_cutoff, len(relevant_items))
results_current_cutoff[EvaluatorMetrics.RECALL.value] += recall(is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NDCG.value] += ndcg(recommended_items_current_cutoff, relevant_items, relevance=self.get_user_test_ratings(test_user), at=cutoff)
results_current_cutoff[EvaluatorMetrics.HIT_RATE.value] += is_relevant_current_cutoff.sum()
results_current_cutoff[EvaluatorMetrics.ARHR.value] += arhr(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.RMSE.value] += user_rmse
results_current_cutoff[EvaluatorMetrics.MRR.value].add_recommendations(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.MAP.value].add_recommendations(is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NOVELTY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.AVERAGE_POPULARITY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_GINI.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.SHANNON_ENTROPY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_ITEM.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_USER.value].add_recommendations(recommended_items_current_cutoff, test_user)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_MEAN_INTER_LIST.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_HERFINDAHL.value].add_recommendations(recommended_items_current_cutoff)
if EvaluatorMetrics.DIVERSITY_SIMILARITY.value in results_current_cutoff:
results_current_cutoff[EvaluatorMetrics.DIVERSITY_SIMILARITY.value].add_recommendations(recommended_items_current_cutoff)
if time.time() - start_time_print > 30 or n_users_evaluated==len(self.usersToEvaluate):
elapsed_time = time.time()-start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(elapsed_time)
print("{}: Processed {} ( {:.2f}% ) in {:.2f} {}. Users per second: {:.0f}".format(
self.EVALUATOR_NAME,
n_users_evaluated,
100.0* float(n_users_evaluated)/len(self.usersToEvaluate),
new_time_value, new_time_unit,
float(n_users_evaluated)/elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_print = time.time()
return results_dict, n_users_evaluated
def run_fit(self):
# Display ConvergenceWarning only once and not for every item it occurs
warnings.simplefilter("once", category=ConvergenceWarning)
# initialize the ElasticNet model
self.model = ElasticNet(alpha=1e-4,
l1_ratio=self.l1_ratio,
positive=self.positive_only,
fit_intercept=False,
copy_X=False,
precompute=True,
selection='random',
max_iter=100,
tol=1e-4)
URM_train = check_matrix(self.URM, 'csc', dtype=np.float32)
n_items = URM_train.shape[1]
# Use array as it reduces memory requirements compared to lists
dataBlock = 10000000
rows = np.zeros(dataBlock, dtype=np.int32)
cols = np.zeros(dataBlock, dtype=np.int32)
values = np.zeros(dataBlock, dtype=np.float32)
numCells = 0
start_time = time.time()
start_time_printBatch = start_time
# fit each item's factors sequentially (not in parallel)
for currentItem in range(n_items):
# get the target column
y = URM_train[:, currentItem].toarray()
if y.sum() == 0.0:
continue
# set the j-th column of X to zero
start_pos = URM_train.indptr[currentItem]
end_pos = URM_train.indptr[currentItem + 1]
current_item_data_backup = URM_train.data[start_pos:end_pos].copy()
URM_train.data[start_pos:end_pos] = 0.0
# fit one ElasticNet model per column
self.model.fit(URM_train, y)
nonzero_model_coef_index = self.model.sparse_coef_.indices
nonzero_model_coef_value = self.model.sparse_coef_.data
local_topK = min(len(nonzero_model_coef_value) - 1, self.topK)
relevant_items_partition = (
-nonzero_model_coef_value
).argpartition(local_topK)[0:local_topK]
relevant_items_partition_sorting = np.argsort(
-nonzero_model_coef_value[relevant_items_partition])
ranking = relevant_items_partition[
relevant_items_partition_sorting]
for index in range(len(ranking)):
if numCells == len(rows):
rows = np.concatenate(
(rows, np.zeros(dataBlock, dtype=np.int32)))
cols = np.concatenate(
(cols, np.zeros(dataBlock, dtype=np.int32)))
values = np.concatenate(
(values, np.zeros(dataBlock, dtype=np.float32)))
rows[numCells] = nonzero_model_coef_index[ranking[index]]
cols[numCells] = currentItem
values[numCells] = nonzero_model_coef_value[ranking[index]]
numCells += 1
# finally, replace the original values of the j-th column
URM_train.data[start_pos:end_pos] = current_item_data_backup
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
if time.time(
) - start_time_printBatch > 300 or currentItem == n_items - 1:
print(
"Processed {} ( {:.2f}% ) in {:.2f} {}. Items per second: {:.2f}"
.format(currentItem + 1,
100.0 * float(currentItem + 1) / n_items,
new_time_value, new_time_unit,
float(currentItem) / elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
# generate the sparse weight matrix
self.W_sparse = sps.csr_matrix(
(values[:numCells], (rows[:numCells], cols[:numCells])),
shape=(n_items, n_items),
dtype=np.float32)
def search(
self,
recommender_input_args,
hyperparameter_search_space,
metric_to_optimize=None,
cutoff_to_optimize=None,
n_cases=None,
n_random_starts=None,
output_folder_path=None,
output_file_name_root=None,
save_model="best",
save_metadata=True,
resume_from_saved=False,
recommender_input_args_last_test=None,
evaluate_on_test="best",
max_total_time=None,
terminate_on_memory_error=True,
):
"""
:param recommender_input_args:
:param hyperparameter_search_space:
:param metric_to_optimize:
:param cutoff_to_optimize:
:param n_cases:
:param n_random_starts:
:param output_folder_path:
:param output_file_name_root:
:param save_model: "no" don't save anything
"all" save every model
"best" save the best model trained on train data alone and on last, if present
"last" save only last, if present
:param save_metadata:
:param recommender_input_args_last_test:
:return:
"""
### default hyperparameters for BayesianSkopt are set here
self._set_skopt_params()
self._set_search_attributes(
recommender_input_args, recommender_input_args_last_test,
hyperparameter_search_space.keys(), metric_to_optimize,
cutoff_to_optimize, output_folder_path, output_file_name_root,
resume_from_saved, save_metadata, save_model, evaluate_on_test,
n_cases, terminate_on_memory_error)
self.n_random_starts = n_random_starts
self.n_calls = n_cases
self.n_jobs = 1
self.n_loaded_counter = 0
self.max_total_time = max_total_time
if self.max_total_time is not None:
total_time_value, total_time_unit = seconds_to_biggest_unit(
self.max_total_time)
self._print(
"{}: The search has a maximum allotted time of {:.2f} {}".
format(self.ALGORITHM_NAME, total_time_value, total_time_unit))
self.hyperparams = dict()
self.hyperparams_names = list()
self.hyperparams_values = list()
skopt_types = [Real, Integer, Categorical]
for name, hyperparam in hyperparameter_search_space.items():
if any(
isinstance(hyperparam, sko_type)
for sko_type in skopt_types):
self.hyperparams_names.append(name)
self.hyperparams_values.append(hyperparam)
self.hyperparams[name] = hyperparam
else:
raise ValueError(
"{}: Unexpected hyperparameter type: {} - {}".format(
self.ALGORITHM_NAME, str(name), str(hyperparam)))
try:
if self.resume_from_saved:
hyperparameters_list_input, result_on_validation_list_saved = self._resume_from_saved(
)
self.x0 = hyperparameters_list_input
self.y0 = result_on_validation_list_saved
self.n_loaded_counter = self.model_counter
if self.n_calls - self.model_counter > 0:
# When resuming an incomplete search the gp_minimize will continue to tell you "Evaluating function at random point" instead
# of "Searching for the next optimal point". This may be due to a bug in the print rather than the underlying process
# https://github.com/scikit-optimize/scikit-optimize/issues/949
self.result = gp_minimize(
self._objective_function_list_input,
self.hyperparams_values,
base_estimator=None,
n_calls=max(0, self.n_calls - self.model_counter),
n_initial_points=max(
0, self.n_random_starts - self.model_counter),
initial_point_generator="random",
acq_func=self.acq_func,
acq_optimizer=self.acq_optimizer,
x0=self.x0,
y0=self.y0,
random_state=self.random_state,
verbose=self.verbose,
callback=None,
n_points=self.n_point,
n_restarts_optimizer=self.n_restarts_optimizer,
xi=self.xi,
kappa=self.kappa,
noise=self.noise,
n_jobs=self.n_jobs)
except ValueError as e:
self._write_log(
"{}: Search interrupted due to ValueError. The evaluated configurations may have had all the same value.\n"
.format(self.ALGORITHM_NAME))
return
except NoValidConfigError as e:
self._write_log("{}: Search interrupted. {}\n".format(
self.ALGORITHM_NAME, e))
return
except TimeoutError as e:
# When in TimeoutError, stop search but continue to train the _last model, if requested
self._write_log("{}: Search interrupted. {}\n".format(
self.ALGORITHM_NAME, e))
if self.n_loaded_counter < self.model_counter:
self._write_log(
"{}: Search complete. Best config is {}: {}\n".format(
self.ALGORITHM_NAME,
self.metadata_dict["hyperparameters_best_index"],
self.metadata_dict["hyperparameters_best"]))
if self.recommender_input_args_last_test is not None:
self._evaluate_on_test_with_data_last()
def compute_similarity(self, start_col=None, end_col=None, block_size = 100):
"""
Compute the similarity for the given dataset
:param self:
:param start_col: column to begin with
:param end_col: column to stop before, end_col is excluded
:return:
"""
similarity_builder = Incremental_Similarity_Builder(self.n_columns, initial_data_block=self.n_columns*self.topK, dtype = np.float32)
start_time = time.time()
start_time_print_batch = start_time
processed_items = 0
if self.adjusted_cosine:
self.applyAdjustedCosine()
elif self.pearson_correlation:
self.applyPearsonCorrelation()
elif self.tanimoto_coefficient or self.dice_coefficient or self.tversky_coefficient:
self.useOnlyBooleanInteractions()
# We explore the matrix column-wise
self.dataMatrix = check_matrix(self.dataMatrix, 'csc')
# Compute sum of squared values to be used in normalization
sum_of_squared = np.array(self.dataMatrix.power(2).sum(axis=0)).ravel()
# Tanimoto does not require the square root to be applied
if not (self.tanimoto_coefficient or self.dice_coefficient or self.tversky_coefficient):
sum_of_squared = np.sqrt(sum_of_squared)
if self.asymmetric_cosine:
sum_of_squared_to_alpha = np.power(sum_of_squared + 1e-6, 2 * self.asymmetric_alpha)
sum_of_squared_to_1_minus_alpha = np.power(sum_of_squared + 1e-6, 2 * (1 - self.asymmetric_alpha))
self.dataMatrix = check_matrix(self.dataMatrix, 'csc')
start_col_local = 0
end_col_local = self.n_columns
if start_col is not None and start_col>0 and start_col<self.n_columns:
start_col_local = start_col
if end_col is not None and end_col>start_col_local and end_col<self.n_columns:
end_col_local = end_col
start_col_block = start_col_local
this_block_size = 0
# Compute all similarities for each item using vectorization
while start_col_block < end_col_local:
# Compute block first and last column
end_col_block = min(start_col_block + block_size, end_col_local)
this_block_size = end_col_block-start_col_block
# All data points for a given item
item_data = self.dataMatrix[:, start_col_block:end_col_block]
item_data = item_data.toarray()
# Compute item similarities
if self.use_row_weights:
this_block_weights = self.dataMatrix_weighted.T.dot(item_data)
else:
this_block_weights = self.dataMatrix.T.dot(item_data)
for col_index_in_block in range(this_block_size):
if this_block_size == 1:
this_column_weights = this_block_weights.ravel()
else:
this_column_weights = this_block_weights[:,col_index_in_block]
columnIndex = col_index_in_block + start_col_block
this_column_weights[columnIndex] = 0.0
# Apply normalization and shrinkage, ensure denominator != 0
if self.normalize:
if self.asymmetric_cosine:
denominator = sum_of_squared_to_alpha[columnIndex] * sum_of_squared_to_1_minus_alpha + self.shrink + 1e-6
else:
denominator = sum_of_squared[columnIndex] * sum_of_squared + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights, 1 / denominator)
# Apply the specific denominator for Tanimoto
elif self.tanimoto_coefficient:
denominator = sum_of_squared[columnIndex] + sum_of_squared - this_column_weights + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights, 1 / denominator)
elif self.dice_coefficient:
denominator = sum_of_squared[columnIndex] + sum_of_squared + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights, 1 / denominator)
elif self.tversky_coefficient:
denominator = this_column_weights + \
(sum_of_squared[columnIndex] - this_column_weights)*self.tversky_alpha + \
(sum_of_squared - this_column_weights)*self.tversky_beta + self.shrink + 1e-6
this_column_weights = np.multiply(this_column_weights, 1 / denominator)
# If no normalization or tanimoto is selected, apply only shrink
elif self.shrink != 0:
this_column_weights = this_column_weights/self.shrink
# Sort indices and select topK
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
relevant_items_partition = (-this_column_weights).argpartition(self.topK - 1)[0:self.topK]
relevant_items_partition_sorting = np.argsort(-this_column_weights[relevant_items_partition])
top_k_idx = relevant_items_partition[relevant_items_partition_sorting]
# Incrementally build sparse matrix, do not add zeros
notZerosMask = this_column_weights[top_k_idx] != 0.0
numNotZeros = np.sum(notZerosMask)
similarity_builder.add_data_lists(row_list_to_add=top_k_idx[notZerosMask],
col_list_to_add=np.ones(numNotZeros) * columnIndex,
data_list_to_add=this_column_weights[top_k_idx][notZerosMask])
# Add previous block size
start_col_block += this_block_size
processed_items += this_block_size
if time.time() - start_time_print_batch >= 300 or end_col_block==end_col_local:
column_per_sec = processed_items / (time.time() - start_time + 1e-9)
new_time_value, new_time_unit = seconds_to_biggest_unit(time.time() - start_time)
print("Similarity column {} ({:4.1f}%), {:.2f} column/sec. Elapsed time {:.2f} {}".format(
processed_items, processed_items / (end_col_local - start_col_local) * 100, column_per_sec, new_time_value, new_time_unit))
sys.stdout.flush()
sys.stderr.flush()
start_time_print_batch = time.time()
# End while on columns
W_sparse = similarity_builder.get_SparseMatrix()
return W_sparse
def _train_with_early_stopping(self, epochs_max, epochs_min=0,
validation_every_n=None, stop_on_validation=False,
validation_metric=None, lower_validations_allowed=None, evaluator_object=None,
algorithm_name="Incremental_Training_Early_Stopping"):
start_time = time.time()
self.best_validation_metric = None
lower_validatons_count = 0
convergence = False
self.epochs_best = 0
epochs_current = 0
while epochs_current < epochs_max and not convergence:
self._run_epoch(epochs_current)
# If no validation required, always keep the latest
if evaluator_object is None:
self.epochs_best = epochs_current
# Determine whether a validaton step is required
elif (epochs_current + 1) % validation_every_n == 0:
print("{}: Validation begins...".format(algorithm_name))
self._prepare_model_for_validation()
# If the evaluator validation has multiple cutoffs, choose the first one
results_run, results_run_string = evaluator_object.evaluateRecommender(self)
results_run = results_run[list(results_run.keys())[0]]
print("{}: {}".format(algorithm_name, results_run_string))
# Update optimal model
current_metric_value = results_run[validation_metric]
if not np.isfinite(current_metric_value):
if isinstance(self, BaseTempFolder):
# If the recommender uses BaseTempFolder, clean the temp folder
self._clean_temp_folder(temp_file_folder=self.temp_file_folder)
assert False, "{}: metric value is not a finite number, terminating!".format(RECOMMENDER_NAME)
if self.best_validation_metric is None or self.best_validation_metric < current_metric_value:
print("{}: New best model found! Updating.".format(algorithm_name))
self.best_validation_metric = current_metric_value
self._update_best_model()
self.epochs_best = epochs_current +1
lower_validatons_count = 0
else:
lower_validatons_count += 1
if stop_on_validation and lower_validatons_count >= lower_validations_allowed and epochs_current >= epochs_min:
convergence = True
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(elapsed_time)
print("{}: Convergence reached! Terminating at epoch {}. Best value for '{}' at epoch {} is {:.4f}. Elapsed time {:.2f} {}".format(
algorithm_name, epochs_current+1, validation_metric, self.epochs_best, self.best_validation_metric, new_time_value, new_time_unit))
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(elapsed_time)
print("{}: Epoch {} of {}. Elapsed time {:.2f} {}".format(
algorithm_name, epochs_current+1, epochs_max, new_time_value, new_time_unit))
epochs_current += 1
sys.stdout.flush()
sys.stderr.flush()
# If no validation required, keep the latest
if evaluator_object is None:
self._prepare_model_for_validation()
self._update_best_model()
# Stop when max epochs reached and not early-stopping
if not convergence:
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(elapsed_time)
if evaluator_object is not None and self.best_validation_metric is not None:
print("{}: Terminating at epoch {}. Best value for '{}' at epoch {} is {:.4f}. Elapsed time {:.2f} {}".format(
algorithm_name, epochs_current, validation_metric, self.epochs_best, self.best_validation_metric, new_time_value, new_time_unit))
else:
print("{}: Terminating at epoch {}. Elapsed time {:.2f} {}".format(
algorithm_name, epochs_current, new_time_value, new_time_unit))
def fit(self, l1_ratio=0.1, alpha=1.0, positive_only=True, topK=100):
assert l1_ratio >= 0 and l1_ratio <= 1, "{}: l1_ratio must be between 0 and 1, provided value was {}".format(
self.RECOMMENDER_NAME, l1_ratio)
self.l1_ratio = l1_ratio
self.positive_only = positive_only
self.topK = topK
# initialize the ElasticNet model
self.model = ElasticNet(alpha=alpha,
l1_ratio=self.l1_ratio,
positive=self.positive_only,
fit_intercept=False,
copy_X=False,
precompute=True,
selection='random',
max_iter=100,
tol=1e-4)
URM_train = check_matrix(self.URM_train, 'csc', dtype=np.float32)
n_items = URM_train.shape[1]
similarity_builder = Incremental_Similarity_Builder(
self.n_items,
initial_data_block=self.n_items * self.topK,
dtype=np.float32)
start_time = time.time()
start_time_printBatch = start_time
# fit each item's factors sequentially (not in parallel)
for currentItem in range(n_items):
# get the target column
y = URM_train[:, currentItem].toarray()
# set the j-th column of X to zero
start_pos = URM_train.indptr[currentItem]
end_pos = URM_train.indptr[currentItem + 1]
current_item_data_backup = URM_train.data[start_pos:end_pos].copy()
URM_train.data[start_pos:end_pos] = 0.0
# fit one ElasticNet model per column
self.model.fit(URM_train, y)
# self.model.coef_ contains the coefficient of the ElasticNet model
# let's keep only the non-zero values
# Select topK values
# Sorting is done in three steps. Faster then plain np.argsort for higher number of items
# - Partition the data to extract the set of relevant items
# - Sort only the relevant items
# - Get the original item index
nonzero_model_coef_index = self.model.sparse_coef_.indices
nonzero_model_coef_value = self.model.sparse_coef_.data
local_topK = min(len(nonzero_model_coef_value) - 1, self.topK)
relevant_items_partition = (
-nonzero_model_coef_value
).argpartition(local_topK)[0:local_topK]
relevant_items_partition_sorting = np.argsort(
-nonzero_model_coef_value[relevant_items_partition])
ranking = relevant_items_partition[
relevant_items_partition_sorting]
similarity_builder.add_data_lists(
row_list_to_add=nonzero_model_coef_index[ranking],
col_list_to_add=np.ones(len(nonzero_model_coef_index)) *
currentItem,
data_list_to_add=nonzero_model_coef_value[ranking])
# finally, replace the original values of the j-th column
URM_train.data[start_pos:end_pos] = current_item_data_backup
elapsed_time = time.time() - start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(
elapsed_time)
if time.time(
) - start_time_printBatch > 300 or currentItem == n_items - 1:
self._print(
"Processed {} ({:4.1f}%) in {:.2f} {}. Items per second: {:.2f}"
.format(currentItem + 1,
100.0 * float(currentItem + 1) / n_items,
new_time_value, new_time_unit,
float(currentItem) / elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
self.W_sparse = similarity_builder.get_SparseMatrix()
def evaluateRecommender(self, recommender_object):
"""
:param recommender_object: the trained recommender object, a BaseRecommender subclass
:param URM_test_list: list of URMs to test the recommender against, or a single URM object
:param cutoff_list: list of cutoffs to be use to report the scores, or a single cutoff
"""
results_dict = {}
for cutoff in self.cutoff_list:
results_dict[cutoff] = create_empty_metrics_dict(self.n_items, self.n_users,
recommender_object.URM_train,
self.ignore_items_ID,
self.ignore_users_ID,
cutoff,
self.diversity_object)
start_time = time.time()
start_time_print = time.time()
n_users_evaluated = 0
if self.ignore_items_flag:
recommender_object.set_items_to_ignore(self.ignore_items_ID)
for test_user in self.usersToEvaluate:
# Being the URM CSR, the indices are the non-zero column indexes
relevant_items = self.get_user_relevant_items(test_user)
relevant_items_rating = self.get_user_test_ratings(test_user)
n_users_evaluated += 1
items_to_compute = self._get_user_specific_items_to_compute(test_user)
recommended_items, all_items_predicted_ratings = recommender_object.recommend(np.atleast_1d(test_user),
remove_seen_flag=self.exclude_seen,
cutoff = self.max_cutoff,
remove_top_pop_flag=False,
items_to_compute = items_to_compute,
remove_CustomItems_flag=self.ignore_items_flag,
return_scores = True
)
assert len(recommended_items) == 1, "{}: recommended_items contained recommendations for {} users, expected was {}".format(
self.EVALUATOR_NAME, len(recommended_items), 1)
assert all_items_predicted_ratings.shape[0] == 1, "{}: all_items_predicted_ratings contained scores for {} users, expected was {}".format(
self.EVALUATOR_NAME, all_items_predicted_ratings.shape[0], 1)
assert all_items_predicted_ratings.shape[1] == self.n_items, "{}: all_items_predicted_ratings contained scores for {} items, expected was {}".format(
self.EVALUATOR_NAME, all_items_predicted_ratings.shape[1], self.n_items)
recommended_items = np.array(recommended_items[0])
user_rmse = rmse(all_items_predicted_ratings[0], relevant_items, relevant_items_rating)
recommender_object.reset_items_to_ignore()
is_relevant = np.in1d(recommended_items, relevant_items, assume_unique=True)
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
is_relevant_current_cutoff = is_relevant[0:cutoff]
recommended_items_current_cutoff = recommended_items[0:cutoff]
results_current_cutoff[EvaluatorMetrics.ROC_AUC.value] += roc_auc(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.PRECISION.value] += precision(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.PRECISION_RECALL_MIN_DEN.value] += precision_recall_min_denominator(is_relevant_current_cutoff, len(relevant_items))
results_current_cutoff[EvaluatorMetrics.RECALL.value] += recall(is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NDCG.value] += ndcg(recommended_items_current_cutoff, relevant_items, relevance=self.get_user_test_ratings(test_user), at=cutoff)
results_current_cutoff[EvaluatorMetrics.HIT_RATE.value] += is_relevant_current_cutoff.sum()
results_current_cutoff[EvaluatorMetrics.ARHR.value] += arhr(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.RMSE.value] += user_rmse
results_current_cutoff[EvaluatorMetrics.MRR.value].add_recommendations(is_relevant_current_cutoff)
results_current_cutoff[EvaluatorMetrics.MAP.value].add_recommendations(is_relevant_current_cutoff, relevant_items)
results_current_cutoff[EvaluatorMetrics.NOVELTY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.AVERAGE_POPULARITY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_GINI.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.SHANNON_ENTROPY.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_ITEM.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.COVERAGE_USER.value].add_recommendations(recommended_items_current_cutoff, test_user)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_MEAN_INTER_LIST.value].add_recommendations(recommended_items_current_cutoff)
results_current_cutoff[EvaluatorMetrics.DIVERSITY_HERFINDAHL.value].add_recommendations(recommended_items_current_cutoff)
if EvaluatorMetrics.DIVERSITY_SIMILARITY.value in results_current_cutoff:
results_current_cutoff[EvaluatorMetrics.DIVERSITY_SIMILARITY.value].add_recommendations(recommended_items_current_cutoff)
if time.time() - start_time_print > 30 or n_users_evaluated==len(self.usersToEvaluate):
elapsed_time = time.time()-start_time
new_time_value, new_time_unit = seconds_to_biggest_unit(elapsed_time)
print("{}: Processed {} ( {:.2f}% ) in {:.2f} {}. Users per second: {:.0f}".format(
self.EVALUATOR_NAME,
n_users_evaluated,
100.0* float(n_users_evaluated)/len(self.usersToEvaluate),
new_time_value, new_time_unit,
float(n_users_evaluated)/elapsed_time))
sys.stdout.flush()
sys.stderr.flush()
start_time_print = time.time()
if (n_users_evaluated > 0):
for cutoff in self.cutoff_list:
results_current_cutoff = results_dict[cutoff]
for key in results_current_cutoff.keys():
value = results_current_cutoff[key]
if isinstance(value, Metrics_Object):
results_current_cutoff[key] = value.get_metric_value()
else:
results_current_cutoff[key] = value/n_users_evaluated
precision_ = results_current_cutoff[EvaluatorMetrics.PRECISION.value]
recall_ = results_current_cutoff[EvaluatorMetrics.RECALL.value]
if precision_ + recall_ != 0:
# F1 micro averaged: http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.104.8244&rep=rep1&type=pdf
results_current_cutoff[EvaluatorMetrics.F1.value] = 2 * (precision_ * recall_) / (precision_ + recall_)
else:
print("WARNING: No users had a sufficient number of relevant items")
if self.ignore_items_flag:
recommender_object.reset_items_to_ignore()
results_run_string = get_result_string(results_dict)
return (results_dict, results_run_string)
def fit(self,
topK=100,
alpha=1.,
min_rating=0,
implicit=False,
normalize_similarity=False):
self.topK = topK
self.alpha = alpha
self.min_rating = min_rating
self.implicit = implicit
self.normalize_similarity = normalize_similarity
#
# if X.dtype != np.float32:
# print("P3ALPHA fit: For memory usage reasons, we suggest to use np.float32 as dtype for the dataset")
if self.min_rating > 0:
self.URM_train.data[self.URM_train.data < self.min_rating] = 0
self.URM_train.eliminate_zeros()
if self.implicit:
self.URM_train.data = np.ones(self.URM_train.data.size,
dtype=np.float32)
#Pui is the row-normalized urm
Pui = normalize(self.URM_train, norm='l1', axis=1)
#Piu is the column-normalized, "boolean" urm transposed
X_bool = self.URM_train.transpose(copy=True)
X_bool.data = np.ones(X_bool.data.size, np.float32)
#ATTENTION: axis is still 1 because i transposed before the normalization
Piu = normalize(X_bool, norm='l1', axis=1)
del (X_bool)
# Alfa power
if self.alpha != 1.:
Pui = Pui.power(self.alpha)
Piu = Piu.power(self.alpha)
# Final matrix is computed as Pui * Piu * Pui
# Multiplication unpacked for memory usage reasons
block_dim = 200
d_t = Piu
similarity_builder = Incremental_Similarity_Builder(
Pui.shape[1],
initial_data_block=Pui.shape[1] * self.topK,
dtype=np.float32)
start_time = time.time()
start_time_printBatch = start_time
for current_block_start_row in range(0, Pui.shape[1], block_dim):
if current_block_start_row + block_dim > Pui.shape[1]:
block_dim = Pui.shape[1] - current_block_start_row
similarity_block = d_t[
current_block_start_row:current_block_start_row +
block_dim, :] * Pui
similarity_block = similarity_block.toarray()
for row_in_block in range(block_dim):
row_data = similarity_block[row_in_block, :]
row_data[current_block_start_row + row_in_block] = 0
best = row_data.argsort()[::-1][:self.topK]
notZerosMask = row_data[best] != 0.0
values_to_add = row_data[best][notZerosMask]
cols_to_add = best[notZerosMask]
similarity_builder.add_data_lists(
row_list_to_add=np.ones(len(values_to_add)) *
(current_block_start_row + row_in_block),
col_list_to_add=cols_to_add,
data_list_to_add=values_to_add)
if time.time() - start_time_printBatch > 300:
new_time_value, new_time_unit = seconds_to_biggest_unit(
time.time() - start_time)
self._print(
"Similarity column {} ({:4.1f}%), {:.2f} column/sec. Elapsed time {:.2f} {}"
.format(
current_block_start_row + block_dim,
100.0 * float(current_block_start_row + block_dim) /
Pui.shape[1],
float(current_block_start_row + block_dim) /
(time.time() - start_time), new_time_value,
new_time_unit))
sys.stdout.flush()
sys.stderr.flush()
start_time_printBatch = time.time()
self.W_sparse = similarity_builder.get_SparseMatrix()
if self.normalize_similarity:
self.W_sparse = normalize(self.W_sparse, norm='l1', axis=1)
if self.topK != False:
self.W_sparse = similarityMatrixTopK(self.W_sparse, k=self.topK)
self.W_sparse = check_matrix(self.W_sparse, format='csr')
