def test_randomizedsearchcv_best_estimator():
"""Ensure that the best estimator is the one that gives the best score (by
re-running it)"""
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
param_distributions = {
'n_epochs': [5],
'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.04, 0.02),
'n_factors': [1],
'init_std_dev': [0]
}
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae'],
cv=PredefinedKFold(),
joblib_verbose=100)
rs.fit(data)
best_estimator = rs.best_estimator['mae']
# recompute MAE of best_estimator
mae = cross_validate(best_estimator,
data,
measures=['MAE'],
cv=PredefinedKFold())['test_mae']
assert mae == rs.best_score['mae']
def test_randomizedsearchcv_same_splits():
"""Ensure that all parameter combinations are tested on the same splits (we
check their RMSE scores are the same once averaged over the splits, which
should be enough). We use as much parallelism as possible."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, reader=Reader('ml-100k'))
kf = KFold(3, shuffle=True, random_state=4)
# all RMSE should be the same (as param combinations are the same)
param_distributions = {'n_epochs': [5], 'lr_all': uniform(.2, 0),
'reg_all': uniform(.4, 0), 'n_factors': [5],
'random_state': [0]}
rs = RandomizedSearchCV(SVD, param_distributions, measures=['RMSE'], cv=kf,
n_jobs=-1)
rs.fit(data)
rmse_scores = [m for m in rs.cv_results['mean_test_rmse']]
assert len(set(rmse_scores)) == 1 # assert rmse_scores are all equal
# Note: actually, even when setting random_state=None in kf, the same folds
# are used because we use product(param_comb, kf.split(...)). However, it's
# needed to have the same folds when calling fit again:
rs.fit(data)
rmse_scores += [m for m in rs.cv_results['mean_test_rmse']]
assert len(set(rmse_scores)) == 1 # assert rmse_scores are all equal
def test_randomizedsearchcv_same_splits():
"""Ensure that all parameter combinations are tested on the same splits (we
check their RMSE scores are the same once averaged over the splits, which
should be enough). We use as much parallelism as possible."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, reader=Reader('ml-100k'),
rating_scale=(1, 5))
kf = KFold(3, shuffle=True, random_state=4)
# all RMSE should be the same (as param combinations are the same)
param_distributions = {'n_epochs': [5], 'lr_all': uniform(.2, 0),
'reg_all': uniform(.4, 0), 'n_factors': [5],
'random_state': [0]}
rs = RandomizedSearchCV(SVD, param_distributions, measures=['RMSE'], cv=kf,
n_jobs=1)
rs.fit(data)
rmse_scores = [m for m in rs.cv_results['mean_test_rmse']]
assert len(set(rmse_scores)) == 1 # assert rmse_scores are all equal
# Note: actually, even when setting random_state=None in kf, the same folds
# are used because we use product(param_comb, kf.split(...)). However, it's
# needed to have the same folds when calling fit again:
rs.fit(data)
rmse_scores += [m for m in rs.cv_results['mean_test_rmse']]
assert len(set(rmse_scores)) == 1 # assert rmse_scores are all equal
def test_randomizedsearchcv_best_estimator(u1_ml100k):
"""Ensure that the best estimator is the one that gives the best score (by
re-running it)"""
param_distributions = {
'n_epochs': [5],
'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.04, 0.02),
'n_factors': [1],
'init_std_dev': [0]
}
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae'],
cv=PredefinedKFold(),
joblib_verbose=100)
rs.fit(u1_ml100k)
best_estimator = rs.best_estimator['mae']
# recompute MAE of best_estimator
mae = cross_validate(best_estimator,
u1_ml100k,
measures=['MAE'],
cv=PredefinedKFold())['test_mae']
assert mae == rs.best_score['mae']
def test_randomizedsearchcv_cv_results():
"""Test the cv_results attribute"""
f = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(f, Reader('ml-100k'))
kf = KFold(3, shuffle=True, random_state=4)
param_distributions = {'n_epochs': [5], 'lr_all': uniform(.2, .3),
'reg_all': uniform(.4, .3), 'n_factors': [5],
'random_state': [0]}
n_iter = 5
rs = RandomizedSearchCV(SVD, param_distributions, n_iter=n_iter,
measures=['RMSE', 'mae'], cv=kf,
return_train_measures=True)
rs.fit(data)
# test keys split*_test_rmse, mean and std dev.
assert rs.cv_results['split0_test_rmse'].shape == (n_iter,)
assert rs.cv_results['split1_test_rmse'].shape == (n_iter,)
assert rs.cv_results['split2_test_rmse'].shape == (n_iter,)
assert rs.cv_results['mean_test_rmse'].shape == (n_iter,)
assert np.allclose(rs.cv_results['mean_test_rmse'],
np.mean([rs.cv_results['split0_test_rmse'],
rs.cv_results['split1_test_rmse'],
rs.cv_results['split2_test_rmse']], axis=0))
assert np.allclose(rs.cv_results['std_test_rmse'],
np.std([rs.cv_results['split0_test_rmse'],
rs.cv_results['split1_test_rmse'],
rs.cv_results['split2_test_rmse']], axis=0))
# test keys split*_train_mae, mean and std dev.
assert rs.cv_results['split0_train_rmse'].shape == (n_iter,)
assert rs.cv_results['split1_train_rmse'].shape == (n_iter,)
assert rs.cv_results['split2_train_rmse'].shape == (n_iter,)
assert rs.cv_results['mean_train_rmse'].shape == (n_iter,)
assert np.allclose(rs.cv_results['mean_train_rmse'],
np.mean([rs.cv_results['split0_train_rmse'],
rs.cv_results['split1_train_rmse'],
rs.cv_results['split2_train_rmse']], axis=0))
assert np.allclose(rs.cv_results['std_train_rmse'],
np.std([rs.cv_results['split0_train_rmse'],
rs.cv_results['split1_train_rmse'],
rs.cv_results['split2_train_rmse']], axis=0))
# test fit and train times dimensions.
assert rs.cv_results['mean_fit_time'].shape == (n_iter,)
assert rs.cv_results['std_fit_time'].shape == (n_iter,)
assert rs.cv_results['mean_test_time'].shape == (n_iter,)
assert rs.cv_results['std_test_time'].shape == (n_iter,)
assert rs.cv_results['params'] is rs.param_combinations
# assert that best parameter in rs.cv_results['rank_test_measure'] is
# indeed the best_param attribute.
best_index = np.argmin(rs.cv_results['rank_test_rmse'])
assert rs.cv_results['params'][best_index] == rs.best_params['rmse']
best_index = np.argmin(rs.cv_results['rank_test_mae'])
assert rs.cv_results['params'][best_index] == rs.best_params['mae']
def test_randomizedsearchcv_cv_results():
"""Test the cv_results attribute"""
f = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(f, Reader('ml-100k'), rating_scale=(1, 5))
kf = KFold(3, shuffle=True, random_state=4)
param_distributions = {'n_epochs': [5], 'lr_all': uniform(.2, .3),
'reg_all': uniform(.4, .3), 'n_factors': [5],
'random_state': [0]}
n_iter = 5
rs = RandomizedSearchCV(SVD, param_distributions, n_iter=n_iter,
measures=['RMSE', 'mae'], cv=kf,
return_train_measures=True)
rs.fit(data)
# test keys split*_test_rmse, mean and std dev.
assert rs.cv_results['split0_test_rmse'].shape == (n_iter,)
assert rs.cv_results['split1_test_rmse'].shape == (n_iter,)
assert rs.cv_results['split2_test_rmse'].shape == (n_iter,)
assert rs.cv_results['mean_test_rmse'].shape == (n_iter,)
assert np.allclose(rs.cv_results['mean_test_rmse'],
np.mean([rs.cv_results['split0_test_rmse'],
rs.cv_results['split1_test_rmse'],
rs.cv_results['split2_test_rmse']], axis=0))
assert np.allclose(rs.cv_results['std_test_rmse'],
np.std([rs.cv_results['split0_test_rmse'],
rs.cv_results['split1_test_rmse'],
rs.cv_results['split2_test_rmse']], axis=0))
# test keys split*_train_mae, mean and std dev.
assert rs.cv_results['split0_train_rmse'].shape == (n_iter,)
assert rs.cv_results['split1_train_rmse'].shape == (n_iter,)
assert rs.cv_results['split2_train_rmse'].shape == (n_iter,)
assert rs.cv_results['mean_train_rmse'].shape == (n_iter,)
assert np.allclose(rs.cv_results['mean_train_rmse'],
np.mean([rs.cv_results['split0_train_rmse'],
rs.cv_results['split1_train_rmse'],
rs.cv_results['split2_train_rmse']], axis=0))
assert np.allclose(rs.cv_results['std_train_rmse'],
np.std([rs.cv_results['split0_train_rmse'],
rs.cv_results['split1_train_rmse'],
rs.cv_results['split2_train_rmse']], axis=0))
# test fit and train times dimensions.
assert rs.cv_results['mean_fit_time'].shape == (n_iter,)
assert rs.cv_results['std_fit_time'].shape == (n_iter,)
assert rs.cv_results['mean_test_time'].shape == (n_iter,)
assert rs.cv_results['std_test_time'].shape == (n_iter,)
assert rs.cv_results['params'] is rs.param_combinations
# assert that best parameter in rs.cv_results['rank_test_measure'] is
# indeed the best_param attribute.
best_index = np.argmin(rs.cv_results['rank_test_rmse'])
assert rs.cv_results['params'][best_index] == rs.best_params['rmse']
best_index = np.argmin(rs.cv_results['rank_test_mae'])
assert rs.cv_results['params'][best_index] == rs.best_params['mae']
def test_randomizedsearchcv_refit(u1_ml100k):
"""Test refit method of RandomizedSearchCV class."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, Reader('ml-100k'))
param_distributions = {
'n_epochs': [5],
'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.4, 0.2),
'n_factors': [2]
}
# assert rs.fit() and rs.test will use best estimator for mae (first
# appearing in measures)
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=True)
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
mae_preds = rs.best_estimator['mae'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == mae_preds
# assert rs.fit() and rs.test will use best estimator for rmse
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit='rmse')
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
rmse_preds = rs.best_estimator['rmse'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == rmse_preds
# test that predict() can be called
rs.predict(2, 4)
# assert test() and predict() cannot be used when refit is false
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=False)
rs.fit(data)
with pytest.raises(ValueError):
rs.test(data.construct_testset(data.raw_ratings))
with pytest.raises(ValueError):
rs.predict('1', '2')
# test that error is raised if used with load_from_folds
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=True)
with pytest.raises(ValueError):
rs.fit(u1_ml100k)
def hyperparameter_tuning(data):
reader = Reader()
data_rs = Dataset.load_from_df(data, reader)
parameters = {
'n_factors': [50, 100, 200],
'n_epochs': [20, 40],
'lr_all': [0.005, 0.001],
'reg_all': [0.05, 0.02, 0.01]
}
rs = RandomizedSearchCV(SVD, parameters, measures=['rmse'], cv=5)
rs.fit(data_rs)
return rs.best_params['rmse'], rs.best_score
def test_randomizedsearchcv_best_estimator(u1_ml100k):
"""Ensure that the best estimator is the one that gives the best score (by
re-running it)"""
param_distributions = {'n_epochs': [5], 'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.04, 0.02), 'n_factors': [1],
'init_std_dev': [0]}
rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae'],
cv=PredefinedKFold(), joblib_verbose=100)
rs.fit(u1_ml100k)
best_estimator = rs.best_estimator['mae']
# recompute MAE of best_estimator
mae = cross_validate(best_estimator, u1_ml100k, measures=['MAE'],
cv=PredefinedKFold())['test_mae']
assert mae == rs.best_score['mae']
def test_randomizedsearchcv_refit(u1_ml100k):
"""Test refit method of RandomizedSearchCV class."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, Reader('ml-100k'),
rating_scale=(1, 5))
param_distributions = {'n_epochs': [5], 'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.4, 0.2), 'n_factors': [2]}
# assert rs.fit() and rs.test will use best estimator for mae (first
# appearing in measures)
rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'],
cv=2, refit=True)
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
mae_preds = rs.best_estimator['mae'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == mae_preds
# assert rs.fit() and rs.test will use best estimator for rmse
rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'],
cv=2, refit='rmse')
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
rmse_preds = rs.best_estimator['rmse'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == rmse_preds
# test that predict() can be called
rs.predict(2, 4)
# assert test() and predict() cannot be used when refit is false
rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'],
cv=2, refit=False)
rs.fit(data)
with pytest.raises(ValueError):
rs.test(data.construct_testset(data.raw_ratings))
with pytest.raises(ValueError):
rs.predict('1', '2')
# test that error is raised if used with load_from_folds
rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'],
cv=2, refit=True)
with pytest.raises(ValueError):
rs.fit(u1_ml100k)
similarity_options = {
'name': ['pearson_baseline'], #it is recommended to use Pearson Baseline
'user_based': [True, False]
}
parameters_distributions = {
'k': np.arange(1, 60, 2),
'min_k': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
'sim_options': similarity_options
}
searchCV = RandomizedSearchCV(current_algo,
parameters_distributions,
n_iter=30,
measures=['rmse'],
n_jobs=12,
cv=5)
searchCV.fit(data)
end = datetime.datetime.now()
print(end - start, "\nEnd.....")
print(searchCV.best_params['rmse'])
#Second iteration
#0.8864
#{'k': 37, 'min_k': 11, 'sim_options': {'name': 'pearson_baseline', 'user_based': False}}
alg = KNNBaseline(k=37,
min_k=11,
sim_options={
'name': 'pearson_baseline',
'user_based': False
})
cross_validate(alg, data, measures=['RMSE'], cv=kf, verbose=True)
#9M45S
}
print('\nRUNNING GRID SEARCH')
print(' "name": ["pearson", "msd", "cosine"]')
print(' "min_support": [2, 3, 4, 5]')
print(' "user_based": [False, True]\n')
param_grid = {"sim_options": sim_options}
start_time = time.time()
# GridSearchCV
gs = RandomizedSearchCV(KNNWithZScore,
param_grid,
measures=["rmse", "mae"],
cv=3)
gs.fit(training_data)
print()
print("RMSE:", gs.best_score["rmse"])
print(gs.best_params["rmse"])
print()
print("MAE:", gs.best_score["mae"])
print(gs.best_params["mae"])
print()
print("Time Taken: ", time.time() - start_time)
print()
# OUTPUT:
"""
RUNNING GRIDSEARCH
"name": ["pearson", "msd", "cosine"]
'bsl_options': baseline_predictor_options,
'k': np.arange(10, 50, 2),
'min_k': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
'sim_options': similarity_options
} #The final grid of parameters for the Randomized search
kf = KFold(n_splits=5, random_state=0) #We set five folds
CV = RandomizedSearchCV(
KNNBaseline,
param_distributions=grid_of_parameters,
n_iter=50,
measures=['rmse'],
cv=kf,
n_jobs=8,
joblib_verbose=10000
) #Hyperparameters tuning with randomized search using 8 threads
CV.fit(data)
end = datetime.datetime.now()
print(end - start)
# print(CV.best_params)
##BEST PARAMETERS
# {'rmse': {'bsl_options': {'method': 'sgd',
# 'learning_rate': 0.002,
# 'n_epochs': 100,
# 'reg': 0.05},
# 'k': 40,
# 'min_k': 12,
# 'sim_options': {'name': 'pearson_baseline', 'user_based': False}}}
current_algo = KNNBaseline(k=40,
min_k=12,
sim_options={
def hyper_tune(self):
"""
Use Surprises RandomizedSearchCV to tune SVD model hyperparameters.
As recommended by https://surprise.readthedocs.io/en/stable/FAQ.html ,
split the data set into an A and B set to allow for unbiased accuracy
evaluation of the tuned parameters.
RandomizedSearchCV is much faster than GridSearchCV when data set is
not small.
Returns
-------
algo : Tuned Surprise algorithm object
Can be used to train and test.
"""
tune_method = self.tune_method
print('Tuning...')
# Seperate data into A and B sets for unbiased accuracy evaluation
raw_ratings = self.data_ml.raw_ratings
# shuffle ratings
random.shuffle(raw_ratings)
# A = 90% of the data, B = 10% of the data
threshold = int(.9 * len(raw_ratings))
A_raw_ratings = raw_ratings[:threshold]
B_raw_ratings = raw_ratings[threshold:]
# make data_ml the set A
data_ml = self.data_ml
data_ml.raw_ratings = A_raw_ratings
# search grid
param_grid = {
'n_factors': [50, 100, 150],
'n_epochs': [30, 50, 70],
'lr_all': [0.002, 0.005, 0.01],
'reg_all': [0.02, 0.1, 0.4, 0.6]
}
gs = RandomizedSearchCV(SVD,
param_grid,
measures=['rmse', 'mae', 'fcp'],
cv=self.n_splits)
# fit
start_time = time.time()
gs.fit(data_ml)
search_time = time.time() - start_time
print("Took {} seconds for search.".format(search_time))
# best score
print('Best score: ' + str(gs.best_score[tune_method]))
# combination of parameters that gave the best score according to the tune_method
print('Best params: ' + str(gs.best_params[tune_method]))
# get resulting algorithm with tuned parameters
algo = gs.best_estimator[tune_method]
# retrain on the whole set A
trainset = data_ml.build_full_trainset()
algo.fit(trainset)
# Compute biased accuracy on A
predictions = algo.test(trainset.build_testset())
print('Biased accuracy:')
accuracy.rmse(predictions)
accuracy.mae(predictions)
accuracy.fcp(predictions)
# Compute unbiased accuracy on B
# make data_ml the set B
testset = data_ml.construct_testset(B_raw_ratings)
predictions = algo.test(testset)
print('Unbiased accuracy:')
accuracy.rmse(predictions)
accuracy.mae(predictions)
accuracy.fcp(predictions)
return algo