def test_dump():
"""Train an algorithm, compute its predictions then dump them.
Ensure that the predictions that are loaded back are the correct ones, and
that the predictions of the dumped algorithm are also equal to the other
ones."""
random.seed(0)
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
pkf = PredefinedKFold()
trainset, testset = next(pkf.split(data))
algo = BaselineOnly()
algo.fit(trainset)
predictions = algo.test(testset)
with tempfile.NamedTemporaryFile() as tmp_file:
dump.dump(tmp_file.name, predictions, algo)
predictions_dumped, algo_dumped = dump.load(tmp_file.name)
predictions_algo_dumped = algo_dumped.test(testset)
assert predictions == predictions_dumped
assert predictions == predictions_algo_dumped
def test_gridsearchcv_best_estimator():
"""Ensure that the best estimator is the one giving the best score (by
re-running it)"""
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
param_grid = {
'n_epochs': [5],
'lr_all': [0.002, 0.005],
'reg_all': [0.4, 0.6],
'n_factors': [1],
'init_std_dev': [0]
}
gs = GridSearchCV(SVD,
param_grid,
measures=['mae'],
cv=PredefinedKFold(),
joblib_verbose=100)
gs.fit(data)
best_estimator = gs.best_estimator['mae']
# recompute MAE of best_estimator
mae = cross_validate(best_estimator,
data,
measures=['MAE'],
cv=PredefinedKFold())['test_mae']
assert mae == gs.best_score['mae']
def test_performances():
"""Test the returned dict. Also do dumping."""
current_dir = os.path.dirname(os.path.realpath(__file__))
folds_files = [(current_dir + '/custom_train',
current_dir + '/custom_test')]
reader = Reader(line_format='user item rating',
sep=' ',
skip_lines=3,
rating_scale=(1, 5))
data = Dataset.load_from_folds(folds_files=folds_files, reader=reader)
algo = NormalPredictor()
tmp_dir = tempfile.mkdtemp() # create tmp dir
with pytest.warns(UserWarning):
performances = evaluate(algo,
data,
measures=['RmSe', 'Mae'],
with_dump=True,
dump_dir=tmp_dir,
verbose=2)
shutil.rmtree(tmp_dir) # remove tmp dir
assert performances['RMSE'] is performances['rmse']
assert performances['MaE'] is performances['mae']
def test_cross_validate():
# First test with a specified CV iterator.
current_dir = os.path.dirname(os.path.realpath(__file__))
folds_files = [(current_dir + '/custom_train',
current_dir + '/custom_test')]
reader = Reader(line_format='user item rating',
sep=' ',
skip_lines=3,
rating_scale=(1, 5))
data = Dataset.load_from_folds(folds_files=folds_files, reader=reader)
algo = NormalPredictor()
pkf = ms.PredefinedKFold()
ret = ms.cross_validate(algo,
data,
measures=['rmse', 'mae'],
cv=pkf,
verbose=1)
# Basically just test that keys (dont) exist as they should
assert len(ret['test_rmse']) == 1
assert len(ret['test_mae']) == 1
assert len(ret['fit_time']) == 1
assert len(ret['test_time']) == 1
assert 'test_fcp' not in ret
assert 'train_rmse' not in ret
assert 'train_mae' not in ret
# Test that 5 fold CV is used when cv=None
# Also check that train_* key exist when return_train_measures is True.
data = Dataset.load_from_file(current_dir + '/custom_dataset', reader)
ret = ms.cross_validate(algo,
data,
measures=['rmse', 'mae'],
cv=None,
return_train_measures=True,
verbose=True)
assert len(ret['test_rmse']) == 5
assert len(ret['test_mae']) == 5
assert len(ret['fit_time']) == 5
assert len(ret['test_time']) == 5
assert len(ret['train_rmse']) == 5
assert len(ret['train_mae']) == 5
def test_PredifinedKFold():
reader = Reader(line_format='user item rating',
sep=' ',
skip_lines=3,
rating_scale=(1, 5))
current_dir = os.path.dirname(os.path.realpath(__file__))
folds_files = [(current_dir + '/custom_train',
current_dir + '/custom_test')]
data = Dataset.load_from_folds(folds_files=folds_files, reader=reader)
# Make sure rating files are read correctly
pkf = PredefinedKFold()
trainset, testset = next(pkf.split(data))
assert trainset.n_ratings == 6
assert len(testset) == 3
# Make sure pkf returns the same folds as the deprecated data.folds()
with pytest.warns(UserWarning):
trainset_, testset_ = next(data.folds())
assert testset_ == testset
def test_knns():
"""Ensure the k and min_k parameters are effective for knn algorithms."""
# the test and train files are from the ml-100k dataset (10% of u1.base and
# 10 % of u1.test)
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
pkf = PredefinedKFold()
# Actually, as KNNWithMeans and KNNBaseline have back up solutions for when
# there are not enough neighbors, we can't really test them...
klasses = (KNNBasic, ) # KNNWithMeans, KNNBaseline)
k, min_k = 20, 5
for klass in klasses:
algo = klass(k=k, min_k=min_k)
for trainset, testset in pkf.split(data):
algo.fit(trainset)
predictions = algo.test(testset)
for pred in predictions:
if not pred.details['was_impossible']:
assert min_k <= pred.details['actual_k'] <= k
from __future__ import (absolute_import, division, print_function,
unicode_literals)
import os
from idly import CoClustering
from idly import Dataset
from idly import Reader
from idly.model_selection import cross_validate
from idly.model_selection import PredefinedKFold
# the test and train files are from the ml-100k dataset (10% of u1.base and
# 10 % of u1.test)
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k'))
pkf = PredefinedKFold()
def test_CoClustering_parameters():
"""Ensure that all parameters are taken into account."""
# The baseline against which to compare.
algo = CoClustering(n_epochs=1, random_state=1)
rmse_default = cross_validate(algo, data, ['rmse'], pkf)['test_rmse']
# n_cltr_u
algo = CoClustering(n_cltr_u=1, n_epochs=1, random_state=1)
rmse_n_cltr_u = cross_validate(algo, data, ['rmse'], pkf)['test_rmse']
assert rmse_default != rmse_n_cltr_u
def test_randomizedsearchcv_refit():
"""Test refit method of RandomizedSearchCV class."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, Reader('ml-100k'))
param_distributions = {
'n_epochs': [5],
'lr_all': uniform(0.002, 0.003),
'reg_all': uniform(0.4, 0.2),
'n_factors': [2]
}
# assert rs.fit() and rs.test will use best estimator for mae (first
# appearing in measures)
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=True)
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
mae_preds = rs.best_estimator['mae'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == mae_preds
# assert rs.fit() and rs.test will use best estimator for rmse
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit='rmse')
rs.fit(data)
rs_preds = rs.test(data.construct_testset(data.raw_ratings))
rmse_preds = rs.best_estimator['rmse'].test(
data.construct_testset(data.raw_ratings))
assert rs_preds == rmse_preds
# test that predict() can be called
rs.predict(2, 4)
# assert test() and predict() cannot be used when refit is false
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=False)
rs.fit(data)
with pytest.raises(ValueError):
rs.test(data.construct_testset(data.raw_ratings))
with pytest.raises(ValueError):
rs.predict('1', '2')
# test that error is raised if used with load_from_folds
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
rs = RandomizedSearchCV(SVD,
param_distributions,
measures=['mae', 'rmse'],
cv=2,
refit=True)
with pytest.raises(ValueError):
rs.fit(data)
def test_gridsearchcv_refit():
"""Test refit function of GridSearchCV."""
data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_file(data_file, Reader('ml-100k'))
param_grid = {
'n_epochs': [5],
'lr_all': [0.002, 0.005],
'reg_all': [0.4, 0.6],
'n_factors': [2]
}
# assert gs.fit() and gs.test will use best estimator for mae (first
# appearing in measures)
gs = GridSearchCV(SVD,
param_grid,
measures=['mae', 'rmse'],
cv=2,
refit=True)
gs.fit(data)
gs_preds = gs.test(data.construct_testset(data.raw_ratings))
mae_preds = gs.best_estimator['mae'].test(
data.construct_testset(data.raw_ratings))
assert gs_preds == mae_preds
# assert gs.fit() and gs.test will use best estimator for rmse
gs = GridSearchCV(SVD,
param_grid,
measures=['mae', 'rmse'],
cv=2,
refit='rmse')
gs.fit(data)
gs_preds = gs.test(data.construct_testset(data.raw_ratings))
rmse_preds = gs.best_estimator['rmse'].test(
data.construct_testset(data.raw_ratings))
assert gs_preds == rmse_preds
# test that predict() can be called
gs.predict(2, 4)
# assert test() and predict() cannot be used when refit is false
gs = GridSearchCV(SVD,
param_grid,
measures=['mae', 'rmse'],
cv=2,
refit=False)
gs.fit(data)
with pytest.raises(ValueError):
gs_preds = gs.test(data.construct_testset(data.raw_ratings))
with pytest.raises(ValueError):
gs.predict('1', '2')
# test that error is raised if used with load_from_folds
train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train')
test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test')
data = Dataset.load_from_folds([(train_file, test_file)],
Reader('ml-100k'))
gs = GridSearchCV(SVD,
param_grid,
measures=['mae', 'rmse'],
cv=2,
refit=True)
with pytest.raises(ValueError):
gs.fit(data)
def test_wrong_file_name():
"""Ensure file names are checked when creating a (custom) Dataset."""
wrong_files = [('does_not_exist', 'does_not_either')]
with pytest.raises(ValueError):
Dataset.load_from_folds(folds_files=wrong_files, reader=reader)
def test_trainset_testset():
"""Test the construct_trainset and construct_testset methods."""
current_dir = os.path.dirname(os.path.realpath(__file__))
folds_files = [(current_dir + '/custom_train',
current_dir + '/custom_test')]
data = Dataset.load_from_folds(folds_files=folds_files, reader=reader)
with pytest.warns(UserWarning):
trainset, testset = next(data.folds())
# test ur
ur = trainset.ur
assert ur[0] == [(0, 4)]
assert ur[1] == [(0, 4), (1, 2)]
assert ur[40] == [] # not in the trainset
# test ir
ir = trainset.ir
assert ir[0] == [(0, 4), (1, 4), (2, 1)]
assert ir[1] == [(1, 2), (2, 1), (3, 5)]
assert ir[20000] == [] # not in the trainset
# test n_users, n_items, n_ratings, rating_scale
assert trainset.n_users == 4
assert trainset.n_items == 2
assert trainset.n_ratings == 6
assert trainset.rating_scale == (1, 5)
# test raw2inner
for i in range(4):
assert trainset.to_inner_uid('user' + str(i)) == i
with pytest.raises(ValueError):
trainset.to_inner_uid('unkown_user')
for i in range(2):
assert trainset.to_inner_iid('item' + str(i)) == i
with pytest.raises(ValueError):
trainset.to_inner_iid('unkown_item')
# test inner2raw
assert trainset._inner2raw_id_users is None
assert trainset._inner2raw_id_items is None
for i in range(4):
assert trainset.to_raw_uid(i) == 'user' + str(i)
for i in range(2):
assert trainset.to_raw_iid(i) == 'item' + str(i)
assert trainset._inner2raw_id_users is not None
assert trainset._inner2raw_id_items is not None
# Test the build_testset() method
algo = BaselineOnly()
algo.fit(trainset)
testset = trainset.build_testset()
algo.test(testset) # ensure an algorithm can manage the data
assert ('user0', 'item0', 4) in testset
assert ('user3', 'item1', 5) in testset
assert ('user3', 'item1', 0) not in testset
# Test the build_anti_testset() method
algo = BaselineOnly()
algo.fit(trainset)
testset = trainset.build_anti_testset()
algo.test(testset) # ensure an algorithm can manage the data
assert ('user0', 'item0', trainset.global_mean) not in testset
assert ('user3', 'item1', trainset.global_mean) not in testset
assert ('user0', 'item1', trainset.global_mean) in testset
assert ('user3', 'item0', trainset.global_mean) in testset