def getListOfPredictions(self,user,recs_portion, mode='cf'):
out_predictions = []
cbm_value = self.user_database['plays'].mean()
for iid in recs_portion:
if self.user_database[(self.user_database['user'] == user) & (self.user_database['song'] == iid)].shape[0] == 0:
actual_plays = 0
else:
actual_plays = self.user_database[(self.user_database['user'] == user) & (self.user_database['song'] == iid)]['plays'].iloc[0]
if mode == 'cf':
estimated_rating = self.predictions_dict[(user,iid)]
out_predictions.append(Prediction(uid=user, iid=iid, r_ui=actual_plays, est=estimated_rating, details={}))
else:
estimated_rating = cbm_value
out_predictions.append(Prediction(uid=user, iid=iid, r_ui=actual_plays, est=estimated_rating,details={}))
return out_predictions
def two_ensemble_predict(m1_preds, m2_preds, w_1, w_2):
final_preds = []
for x, y in zip(m1_preds, m2_preds):
assert x[0] == y[0]
assert x[1] == y[1]
avg_pred = (w_1 * x[3]) + (w_2 * y[3])
final_preds.append(Prediction(x[0], x[1], x[2], avg_pred, x[4]))
return final_preds
def test_dump():
predictions = [Prediction(None, None, None, None, None)]
algo = AlgoBase()
trainset = Trainset(dict(), *[None] * 8)
with tempfile.NamedTemporaryFile() as tmp_file:
dump(tmp_file.name, predictions, trainset, algo)
def predict(self, uid, iid, r_ui=None, clip=True, verbose=False):
"""
:param uid: user id.
:param iid: item id.
:param r_ui: true rating
:param clip: whether to clip the estimator into the rating scale.
:param verbose: whether to print details of the prediction.
:return: A :obj:`Prediction\
<surprise.prediction_algorithms.predictions.Prediction>` object
containing:
- The (raw) user id ``uid``.
- The (raw) item id ``iid``.
- The true rating ``r_ui`` (:math:`\\hat{r}_{ui}`).
- The estimated rating (:math:`\\hat{r}_{ui}`).
- Some additional details about the prediction that might be useful
for later analysis.
"""
try:
iuid = self.trainset.to_inner_uid(uid)
except ValueError:
iuid = 'UKN__' + str(uid)
try:
iiid = self.trainset.to_inner_iid(iid)
except ValueError:
iiid = 'UKN__' + str(iid)
details = {}
try:
est = self.estimate(iuid, iiid)
# if the details dict was also returned.
if isinstance(est, tuple):
est, details = est
details['was_impossible'] = False
except PredictionImpossible as e:
est = self.default_prediction()
details['was_impossible'] = True
details['reason'] = str(e)
# clip estimate into [lower_bound, higher_bound]
if clip:
lower_bound, higher_bound = self.trainset.rating_scale
est = min(higher_bound, est)
est = max(lower_bound, est)
pred = Prediction(iuid, iiid, r_ui, est, details)
if verbose:
print(pred)
return pred
def three_ensemble_predict(m1_preds, m2_preds, m3_preds, w_1, w_2, w_3):
final_preds = []
for x, y, z in zip(m1_preds, m2_preds, m3_preds):
assert x[0] == y[0] == z[0]
assert x[1] == y[1] == z[1]
# avg_pred = (x[3] + y[3] + z[3]) / 3.0
avg_pred = (w_1 * x[3]) + (w_2 * y[3]) + (w_3 * z[3])
final_preds.append(Prediction(x[0], x[1], x[2], avg_pred, x[4]))
return final_preds
def two_ensemble_kfold_cv(data, m1, m2, w_1, w_2, K=5, random_state=8):
# define a cross-validation iterator
kf = KFold(n_splits=K, random_state=random_state)
rmse_scores = []
m1_scores = []
m2_scores = []
fold_count = 1
for trainset, testset in kf.split(data):
print("Fold #{}...".format(fold_count))
# train and test m1 algorithm.
m1.fit(trainset)
m1_preds = m1.test(testset)
m1_score = accuracy.rmse(m1_preds, verbose=False)
m1_scores.append(m1_score)
print("\tM1 test_rmse={}".format(m1_score))
# train and test item-based m2 algorithm.
m2.fit(trainset)
m2_preds = m2.test(testset)
m2_score = accuracy.rmse(m2_preds, verbose=False)
m2_scores.append(m2_score)
print("\tM2 test_rmse={}".format(m2_score))
final_preds = []
for x, y in zip(m1_preds, m2_preds):
assert x[0] == y[0]
assert x[1] == y[1]
avg_pred = (w_1 * x[3]) + (w_2 * y[3])
final_preds.append(Prediction(x[0], x[1], x[2], avg_pred, x[4]))
# Compute and print Root Mean Squared Error
rmse = accuracy.rmse(final_preds, verbose=False)
print("\tCombined test_rmse={}".format(rmse))
rmse_scores.append(rmse)
fold_count += 1
m1_final = sum(m1_scores) / len(m1_scores)
m2_final = sum(m2_scores) / len(m2_scores)
final_rmse = sum(rmse_scores) / len(rmse_scores)
print("M1 5-fold CV: {}".format(m1_final))
print("M2 5-fold CV: {}".format(m2_final))
print("Ensemble 5-fold CV: {}".format(final_rmse))
return final_rmse
from surprise.prediction_algorithms.predictions import Prediction
import ast
with open(
'{}_seed0_fold{}_predictions.txt'.format(
'predictions/standards/test_ml-1m_GlobalMean_', 0),
'r') as file_handler:
content = ['[' + x.strip('\n') + ']' for x in file_handler.readlines()]
assert (content[0] == '[uid,iid,r_ui,est,details,crossfold_index]')
predictions = [
Prediction(*ast.literal_eval(line)[:-1]) for line in content[1:]
]
print(predictions)
def eval_task(algo,
specific_testsets,
measures,
head_items,
crossfold_index,
save_path=None,
load_path=None,
uid_plus_iid_to_row=None):
"""
Evaluate on specific testsets.
This function exists to make testset evaluation easier to parallelize.
"""
ret = []
if load_path and uid_plus_iid_to_row is None:
tic = time.time()
load_from = '{}_seed0_fold{}_all_predictions.txt'.format(
load_path, crossfold_index)
print('load_from', load_from)
with open(load_from, 'r') as file_handler:
content = [
'[' + x.strip('\n') + ']' for x in file_handler.readlines()
]
assert (content[0] == '[uid,iid,r_ui,est,details,crossfold_index]')
all_predictions = [
Prediction(*ast.literal_eval(line)[:-1])
for line in content[1:]
]
uid_plus_iid_to_row = {}
for prediction in all_predictions:
uid_plus_iid = str(prediction[0]) + '_' + str(prediction[1])
uid_plus_iid_to_row[uid_plus_iid] = prediction
print(
'Loading predictions within eval_task took {}'.format(time.time() -
tic))
for key, specific_testset in specific_testsets.items():
start_specific_testset = time.time()
if uid_plus_iid_to_row:
# if this dict is populated we should use it. if it is empty we can't use it, need to run algo.test
predictions = []
tic = time.time()
if isinstance(specific_testset, np.ndarray):
iterate_on = specific_testset.tolist()
else:
iterate_on = specific_testset
for prediction in iterate_on:
uid_plus_iid = str(prediction[0]) + '_' + str(prediction[1])
predictions.append(uid_plus_iid_to_row[uid_plus_iid])
#print('Took {} seconds to load {} predictions from uid_plus_iid_to_row'.format(time.time() - tic, len(predictions)))
else:
predictions = algo.test(specific_testset)
if save_path and load_path is None and uid_plus_iid_to_row is None: # if you just loaded the predictions, don't save them again, waste of time...
with open(
'{}_seed0_fold{}_{}_predictions.txt'.format(
save_path, crossfold_index, key), 'w') as file_handler:
file_handler.write(
'uid,iid,r_ui,est,details,crossfold_index\n')
for prediction in predictions:
file_handler.write(','.join([str(x) for x in prediction] +
[str(crossfold_index)]) + '\n')
if not predictions:
ret.append([key, {}, 0, 0])
continue
test_measures = {}
for m in measures:
tic = time.time()
eval_func = getattr(accuracy, m.lower())
result = eval_func(predictions, verbose=0)
# NMV 10/26: rewriting this whole chunk b/c we refactored accuracy.py.
#if 'ndcg' in m:
if m == 'list_metrics':
tail_result = eval_func(predictions,
verbose=0,
head_items=head_items)
for metric_name in result.keys():
mean_val, frac_of_users = result[metric_name]
tail_mean_val, tail_frac = tail_result[metric_name]
test_measures[metric_name] = mean_val
test_measures[metric_name + '_frac'] = frac_of_users
test_measures['tail' + metric_name] = tail_mean_val
test_measures['tail' + metric_name + '_frac'] = tail_frac
# sub_measures = m.split('_')
# for i_sm, sub_measure in enumerate(sub_measures):
# mean_val, frac_of_users = result[i_sm]
# tail_mean_val, _ = tail_result[i_sm]
# test_measures[sub_measure] = mean_val
# test_measures[sub_measure + '_frac'] = frac_of_users
# test_measures['tail' + sub_measure] = tail_mean_val
else:
test_measures[m] = result
test_time = time.time() - start_specific_testset
ret.append([key, test_measures, test_time, len(specific_testset)])
return ret
def cross_validate_custom(algo,
nonboycott,
boycott,
boycott_uid_set,
like_boycott_uid_set,
measures=None,
cv=5,
return_train_measures=False,
n_jobs=-1,
pre_dispatch='2*n_jobs',
verbose=False,
head_items=None,
save_path=None,
load_path=None,
load_boycotts_path=None,
data=None):
'''
Run a cross validation procedure for a given algorithm, reporting accuracy
measures and computation times.
See an example in the :ref:`User Guide <cross_validate_example>`.
Args:
algo(:obj:`AlgoBase \
<surprise.prediction_algorithms.algo_base.AlgoBase>`):
The algorithm to evaluate.
data(:obj:`Dataset <surprise.dataset.Dataset>`): The dataset on which
to evaluate the algorithm.
measures(list of string): The performance measures to compute. Allowed
names are function names as defined in the :mod:`accuracy
<surprise.accuracy>` module. Default is ``['rmse', 'mae']``.
cv(cross-validation iterator, int or ``None``): Determines how the
``data`` parameter will be split (i.e. how trainsets and testsets
will be defined). If an int is passed, :class:`KFold
<surprise.model_selection.split.KFold>` is used with the
appropriate ``n_splits`` parameter. If ``None``, :class:`KFold
<surprise.model_selection.split.KFold>` is used with
``n_splits=5``.
return_train_measures(bool): Whether to compute performance measures on
the trainsets. Default is ``False``.
n_jobs(int): The maximum number of folds evaluated in parallel.
- If ``-1``, all CPUs are used.
- If ``1`` is given, no parallel computing code is used at all,\
which is useful for debugging.
- For ``n_jobs`` below ``-1``, ``(n_cpus + n_jobs + 1)`` are\
used. For example, with ``n_jobs = -2`` all CPUs but one are\
used.
Default is ``-1``.
pre_dispatch(int or string): Controls the number of jobs that get
dispatched during parallel execution. Reducing this number can be
useful to avoid an explosion of memory consumption when more jobs
get dispatched than CPUs can process. This parameter can be:
- ``None``, in which case all the jobs are immediately created\
and spawned. Use this for lightweight and fast-running\
jobs, to avoid delays due to on-demand spawning of the\
jobs.
- An int, giving the exact number of total jobs that are\
spawned.
- A string, giving an expression as a function of ``n_jobs``,\
as in ``'2*n_jobs'``.
Default is ``'2*n_jobs'``.
verbose(int): If ``True`` accuracy measures for each split are printed,
as well as train and test times. Averages and standard deviations
over all splits are also reported. Default is ``False``: nothing is
printed.
Returns:
dict: A dict with the following keys:
- ``'test_*'`` where ``*`` corresponds to a lower-case accuracy
measure, e.g. ``'test_rmse'``: numpy array with accuracy values
for each testset.
- ``'train_*'`` where ``*`` corresponds to a lower-case accuracy
measure, e.g. ``'train_rmse'``: numpy array with accuracy values
for each trainset. Only available if ``return_train_measures`` is
``True``.
- ``'fit_time'``: numpy array with the training time in seconds for
each split.
- ``'test_time'``: numpy array with the testing time in seconds for
each split.
'''
measures = [m.lower() for m in measures]
num_folds = cv
cv = KFold(cv, random_state=0)
args_list = []
# IF we're going to parallelize to the folds do this
# number the crossfolds so we can keep track of them when/if they go out to threads
# note that if you're threading at the experiment level this code doesn't do much b/c n_jobs will be set to 1.
# build all the args that will be sent out in parallel
standards = []
if n_jobs != 1:
for (crossfold_index, row) in enumerate(
cv.custom_rating_split(nonboycott, boycott,
{'only': boycott_uid_set},
{'only': like_boycott_uid_set})):
(trainset, nonboycott_testset, boycott_testset,
like_boycott_but_testset, all_like_boycott_testset,
all_testset) = row['only']
args_list += [[
algo, trainset, {
'all': all_testset,
'non-boycott': nonboycott_testset,
'boycott': boycott_testset,
'like-boycott': like_boycott_but_testset,
'all-like-boycott': all_like_boycott_testset
}, measures, return_train_measures, crossfold_index
]]
delayed_gen = (delayed(fit_and_score)(algo, trainset, testsets,
measures, return_train_measures,
crossfold_index, head_items,
save_path, load_boycotts_path)
for algo, trainset, testsets, measures,
return_train_measures, crossfold_index in args_list)
out = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch)(delayed_gen)
# But if we're not parallelizing folds (probably because we parallelized at the experiment level)
# just do everthing in series so we don't use 5x memory for no reason
else:
out = []
# load all the standard results
# if load_path:
# tic = time.time()
# for crossfold_index in range(num_folds):
# load_from = '{}_seed0_fold{}_all_predictions.txt'.format(load_path, crossfold_index)
# print('load_from', load_from)
# with open(load_from, 'r') as file_handler:
# content = ['[' + x.strip('\n') + ']' for x in file_handler.readlines()]
# assert(content[0] == '[uid,iid,r_ui,est,details,crossfold_index]')
# # BAD CODE
# numpyify = False
# if numpyify:
# # for some reason I haven't yet figured out, using this version makes standards_results WAY SLOWER (122 seconds -> 7000 seconds)
# # very bizarre, but using the old way (a list of PRediction NamedTuples instead of structured np array seems fine)
# all_predictions = np.array(
# [Prediction(*(ast.literal_eval(line)[:-2] + [0])) for line in content[1:]],
# dtype=[('uid', 'int32'), ('iid', 'int32'), ('r_ui', float), ('est', float), ('details', bool)]
# )
# else:
# all_predictions = [Prediction(*ast.literal_eval(line)[:-1]) for line in content[1:]]
# for prediction in all_predictions:
# uid_plus_iid = str(prediction[0]) + '_' + str(prediction[1])
# if uid_plus_iid in uid_plus_iid_to_row:
# raise ValueError('Got a duplicate somehow! Fix the saved *_all_predictions.txt files!')
# uid_plus_iid_to_row[uid_plus_iid] = prediction
# print('length of uid_plus_iid_to_row dict: {}'.format(len(uid_plus_iid_to_row)))
# print('Loading predictions for all folds took {}'.format(time.time() - tic))
#print('asizeof uid_plus_iid_to_row in MB')
#print(asizeof.asizeof(uid_plus_iid_to_row) / (1024 ** 2))
for (crossfold_index, row) in enumerate(
cv.custom_rating_split(nonboycott, boycott,
{'only': boycott_uid_set},
{'only': like_boycott_uid_set})):
(trainset, nonboycott_testset, boycott_testset,
like_boycott_but_testset, all_like_boycott_testset,
all_testset) = row['only']
testsets = {
'all': all_testset,
'non-boycott': nonboycott_testset,
'boycott': boycott_testset,
'like-boycott': like_boycott_but_testset,
'all-like-boycott': all_like_boycott_testset
}
#print('About to run fit and score for crossfold index {}'.format(crossfold_index))
#print('psutil.virtual_memory().used {} (GB)'.format(psutil.virtual_memory().used / (1024**3)))
tic = time.time()
results = fit_and_score(algo, trainset, testsets, measures,
return_train_measures, crossfold_index,
head_items, save_path)
#print('fit_and_score for crossfold {} took {} seconds'.format(crossfold_index, time.time() - tic))
out += [results]
# if uid_plus_iid_to_row:
# tic = time.time()
# standards_results = fit_and_score(
# None, None, testsets, measures, return_train_measures, crossfold_index, head_items, uid_plus_iid_to_row=uid_plus_iid_to_row
# )
# print('standards fit_and_score for crossfold {} took {} seconds'.format(crossfold_index, time.time() - tic))
# standards += [standards_results]
if load_path:
for (crossfold_index, row) in enumerate(
cv.custom_rating_split(
data,
Dataset.load_from_df(pd.DataFrame(), reader=Reader()),
{'only': boycott_uid_set},
{'only': like_boycott_uid_set})):
uid_plus_iid_to_row = {}
tic = time.time()
load_from = '{}_seed0_fold{}_all_predictions.txt'.format(
load_path, crossfold_index)
print('load_from', load_from)
with open(load_from, 'r') as file_handler:
content = [
'[' + x.strip('\n') + ']'
for x in file_handler.readlines()
]
assert (content[0] ==
'[uid,iid,r_ui,est,details,crossfold_index]')
numpyify = False
if numpyify:
# for some reason I haven't yet figured out, using this version makes standards_results WAY SLOWER (122 seconds -> 7000 seconds)
# very bizarre, but using the old way (a list of PRediction NamedTuples instead of structured np array seems fine)
all_predictions = np.array([
Prediction(*(ast.literal_eval(line)[:-2] + [0]))
for line in content[1:]
],
dtype=[('uid', 'int32'),
('iid', 'int32'),
('r_ui', float),
('est', float),
('details', bool)])
else:
all_predictions = [
Prediction(*ast.literal_eval(line)[:-1])
for line in content[1:]
]
for prediction in all_predictions:
uid_plus_iid = str(prediction[0]) + '_' + str(
prediction[1])
if uid_plus_iid in uid_plus_iid_to_row:
raise ValueError(
'Got a duplicate somehow! Fix the saved *_all_predictions.txt files!'
)
uid_plus_iid_to_row[uid_plus_iid] = prediction
#print('Loading predictions for fold {} took {}'.format(crossfold_index, time.time() - tic))
(trainset, nonboycott_testset, boycott_testset,
like_boycott_but_testset, all_like_boycott_testset,
all_testset) = row['only']
testsets = {
'all': all_testset,
'non-boycott': nonboycott_testset,
'boycott': boycott_testset,
'like-boycott': like_boycott_but_testset,
'all-like-boycott': all_like_boycott_testset
}
print(
'About to run standards fit and score for crossfold index {}'
.format(crossfold_index))
print('psutil.virtual_memory().used {} (GB)'.format(
psutil.virtual_memory().used / (1024**3)))
tic = time.time()
results = fit_and_score(
None,
None,
testsets,
measures,
return_train_measures,
crossfold_index,
head_items,
save_path,
uid_plus_iid_to_row=uid_plus_iid_to_row)
print(
'standards fit_and_score for crossfold {} took {} seconds'.
format(crossfold_index,
time.time() - tic))
standards += [results]
ret = merge_scores(out, standards)
if verbose:
print(ret)
return ret