def test_dump(): """Train an algorithm, compute its predictions then dump them. Ensure that the predictions that are loaded back are the correct ones, and that the predictions of the dumped algorithm are also equal to the other ones.""" random.seed(0) train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) pkf = PredefinedKFold() trainset, testset = next(pkf.split(data)) algo = BaselineOnly() algo.fit(trainset) predictions = algo.test(testset) with tempfile.NamedTemporaryFile() as tmp_file: dump.dump(tmp_file.name, predictions, algo) predictions_dumped, algo_dumped = dump.load(tmp_file.name) predictions_algo_dumped = algo_dumped.test(testset) assert predictions == predictions_dumped assert predictions == predictions_algo_dumped
def test_gridsearchcv_best_estimator(): """Ensure that the best estimator is the one giving the best score (by re-running it)""" train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) param_grid = { 'n_epochs': [5], 'lr_all': [0.002, 0.005], 'reg_all': [0.4, 0.6], 'n_factors': [1], 'init_std_dev': [0] } gs = GridSearchCV(SVD, param_grid, measures=['mae'], cv=PredefinedKFold(), joblib_verbose=100) gs.fit(data) best_estimator = gs.best_estimator['mae'] # recompute MAE of best_estimator mae = cross_validate(best_estimator, data, measures=['MAE'], cv=PredefinedKFold())['test_mae'] assert mae == gs.best_score['mae']
def test_performances(): """Test the returned dict. Also do dumping.""" current_dir = os.path.dirname(os.path.realpath(__file__)) folds_files = [(current_dir + '/custom_train', current_dir + '/custom_test')] reader = Reader(line_format='user item rating', sep=' ', skip_lines=3, rating_scale=(1, 5)) data = Dataset.load_from_folds(folds_files=folds_files, reader=reader) algo = NormalPredictor() tmp_dir = tempfile.mkdtemp() # create tmp dir with pytest.warns(UserWarning): performances = evaluate(algo, data, measures=['RmSe', 'Mae'], with_dump=True, dump_dir=tmp_dir, verbose=2) shutil.rmtree(tmp_dir) # remove tmp dir assert performances['RMSE'] is performances['rmse'] assert performances['MaE'] is performances['mae']
def test_cross_validate(): # First test with a specified CV iterator. current_dir = os.path.dirname(os.path.realpath(__file__)) folds_files = [(current_dir + '/custom_train', current_dir + '/custom_test')] reader = Reader(line_format='user item rating', sep=' ', skip_lines=3, rating_scale=(1, 5)) data = Dataset.load_from_folds(folds_files=folds_files, reader=reader) algo = NormalPredictor() pkf = ms.PredefinedKFold() ret = ms.cross_validate(algo, data, measures=['rmse', 'mae'], cv=pkf, verbose=1) # Basically just test that keys (dont) exist as they should assert len(ret['test_rmse']) == 1 assert len(ret['test_mae']) == 1 assert len(ret['fit_time']) == 1 assert len(ret['test_time']) == 1 assert 'test_fcp' not in ret assert 'train_rmse' not in ret assert 'train_mae' not in ret # Test that 5 fold CV is used when cv=None # Also check that train_* key exist when return_train_measures is True. data = Dataset.load_from_file(current_dir + '/custom_dataset', reader) ret = ms.cross_validate(algo, data, measures=['rmse', 'mae'], cv=None, return_train_measures=True, verbose=True) assert len(ret['test_rmse']) == 5 assert len(ret['test_mae']) == 5 assert len(ret['fit_time']) == 5 assert len(ret['test_time']) == 5 assert len(ret['train_rmse']) == 5 assert len(ret['train_mae']) == 5
def test_PredifinedKFold(): reader = Reader(line_format='user item rating', sep=' ', skip_lines=3, rating_scale=(1, 5)) current_dir = os.path.dirname(os.path.realpath(__file__)) folds_files = [(current_dir + '/custom_train', current_dir + '/custom_test')] data = Dataset.load_from_folds(folds_files=folds_files, reader=reader) # Make sure rating files are read correctly pkf = PredefinedKFold() trainset, testset = next(pkf.split(data)) assert trainset.n_ratings == 6 assert len(testset) == 3 # Make sure pkf returns the same folds as the deprecated data.folds() with pytest.warns(UserWarning): trainset_, testset_ = next(data.folds()) assert testset_ == testset
def test_knns(): """Ensure the k and min_k parameters are effective for knn algorithms.""" # the test and train files are from the ml-100k dataset (10% of u1.base and # 10 % of u1.test) train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) pkf = PredefinedKFold() # Actually, as KNNWithMeans and KNNBaseline have back up solutions for when # there are not enough neighbors, we can't really test them... klasses = (KNNBasic, ) # KNNWithMeans, KNNBaseline) k, min_k = 20, 5 for klass in klasses: algo = klass(k=k, min_k=min_k) for trainset, testset in pkf.split(data): algo.fit(trainset) predictions = algo.test(testset) for pred in predictions: if not pred.details['was_impossible']: assert min_k <= pred.details['actual_k'] <= k
from __future__ import (absolute_import, division, print_function, unicode_literals) import os from idly import CoClustering from idly import Dataset from idly import Reader from idly.model_selection import cross_validate from idly.model_selection import PredefinedKFold # the test and train files are from the ml-100k dataset (10% of u1.base and # 10 % of u1.test) train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) pkf = PredefinedKFold() def test_CoClustering_parameters(): """Ensure that all parameters are taken into account.""" # The baseline against which to compare. algo = CoClustering(n_epochs=1, random_state=1) rmse_default = cross_validate(algo, data, ['rmse'], pkf)['test_rmse'] # n_cltr_u algo = CoClustering(n_cltr_u=1, n_epochs=1, random_state=1) rmse_n_cltr_u = cross_validate(algo, data, ['rmse'], pkf)['test_rmse'] assert rmse_default != rmse_n_cltr_u
def test_randomizedsearchcv_refit(): """Test refit method of RandomizedSearchCV class.""" data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_file(data_file, Reader('ml-100k')) param_distributions = { 'n_epochs': [5], 'lr_all': uniform(0.002, 0.003), 'reg_all': uniform(0.4, 0.2), 'n_factors': [2] } # assert rs.fit() and rs.test will use best estimator for mae (first # appearing in measures) rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'], cv=2, refit=True) rs.fit(data) rs_preds = rs.test(data.construct_testset(data.raw_ratings)) mae_preds = rs.best_estimator['mae'].test( data.construct_testset(data.raw_ratings)) assert rs_preds == mae_preds # assert rs.fit() and rs.test will use best estimator for rmse rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'], cv=2, refit='rmse') rs.fit(data) rs_preds = rs.test(data.construct_testset(data.raw_ratings)) rmse_preds = rs.best_estimator['rmse'].test( data.construct_testset(data.raw_ratings)) assert rs_preds == rmse_preds # test that predict() can be called rs.predict(2, 4) # assert test() and predict() cannot be used when refit is false rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'], cv=2, refit=False) rs.fit(data) with pytest.raises(ValueError): rs.test(data.construct_testset(data.raw_ratings)) with pytest.raises(ValueError): rs.predict('1', '2') # test that error is raised if used with load_from_folds train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) rs = RandomizedSearchCV(SVD, param_distributions, measures=['mae', 'rmse'], cv=2, refit=True) with pytest.raises(ValueError): rs.fit(data)
def test_gridsearchcv_refit(): """Test refit function of GridSearchCV.""" data_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_file(data_file, Reader('ml-100k')) param_grid = { 'n_epochs': [5], 'lr_all': [0.002, 0.005], 'reg_all': [0.4, 0.6], 'n_factors': [2] } # assert gs.fit() and gs.test will use best estimator for mae (first # appearing in measures) gs = GridSearchCV(SVD, param_grid, measures=['mae', 'rmse'], cv=2, refit=True) gs.fit(data) gs_preds = gs.test(data.construct_testset(data.raw_ratings)) mae_preds = gs.best_estimator['mae'].test( data.construct_testset(data.raw_ratings)) assert gs_preds == mae_preds # assert gs.fit() and gs.test will use best estimator for rmse gs = GridSearchCV(SVD, param_grid, measures=['mae', 'rmse'], cv=2, refit='rmse') gs.fit(data) gs_preds = gs.test(data.construct_testset(data.raw_ratings)) rmse_preds = gs.best_estimator['rmse'].test( data.construct_testset(data.raw_ratings)) assert gs_preds == rmse_preds # test that predict() can be called gs.predict(2, 4) # assert test() and predict() cannot be used when refit is false gs = GridSearchCV(SVD, param_grid, measures=['mae', 'rmse'], cv=2, refit=False) gs.fit(data) with pytest.raises(ValueError): gs_preds = gs.test(data.construct_testset(data.raw_ratings)) with pytest.raises(ValueError): gs.predict('1', '2') # test that error is raised if used with load_from_folds train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) gs = GridSearchCV(SVD, param_grid, measures=['mae', 'rmse'], cv=2, refit=True) with pytest.raises(ValueError): gs.fit(data)
def test_wrong_file_name(): """Ensure file names are checked when creating a (custom) Dataset.""" wrong_files = [('does_not_exist', 'does_not_either')] with pytest.raises(ValueError): Dataset.load_from_folds(folds_files=wrong_files, reader=reader)
def test_trainset_testset(): """Test the construct_trainset and construct_testset methods.""" current_dir = os.path.dirname(os.path.realpath(__file__)) folds_files = [(current_dir + '/custom_train', current_dir + '/custom_test')] data = Dataset.load_from_folds(folds_files=folds_files, reader=reader) with pytest.warns(UserWarning): trainset, testset = next(data.folds()) # test ur ur = trainset.ur assert ur[0] == [(0, 4)] assert ur[1] == [(0, 4), (1, 2)] assert ur[40] == [] # not in the trainset # test ir ir = trainset.ir assert ir[0] == [(0, 4), (1, 4), (2, 1)] assert ir[1] == [(1, 2), (2, 1), (3, 5)] assert ir[20000] == [] # not in the trainset # test n_users, n_items, n_ratings, rating_scale assert trainset.n_users == 4 assert trainset.n_items == 2 assert trainset.n_ratings == 6 assert trainset.rating_scale == (1, 5) # test raw2inner for i in range(4): assert trainset.to_inner_uid('user' + str(i)) == i with pytest.raises(ValueError): trainset.to_inner_uid('unkown_user') for i in range(2): assert trainset.to_inner_iid('item' + str(i)) == i with pytest.raises(ValueError): trainset.to_inner_iid('unkown_item') # test inner2raw assert trainset._inner2raw_id_users is None assert trainset._inner2raw_id_items is None for i in range(4): assert trainset.to_raw_uid(i) == 'user' + str(i) for i in range(2): assert trainset.to_raw_iid(i) == 'item' + str(i) assert trainset._inner2raw_id_users is not None assert trainset._inner2raw_id_items is not None # Test the build_testset() method algo = BaselineOnly() algo.fit(trainset) testset = trainset.build_testset() algo.test(testset) # ensure an algorithm can manage the data assert ('user0', 'item0', 4) in testset assert ('user3', 'item1', 5) in testset assert ('user3', 'item1', 0) not in testset # Test the build_anti_testset() method algo = BaselineOnly() algo.fit(trainset) testset = trainset.build_anti_testset() algo.test(testset) # ensure an algorithm can manage the data assert ('user0', 'item0', trainset.global_mean) not in testset assert ('user3', 'item1', trainset.global_mean) not in testset assert ('user0', 'item1', trainset.global_mean) in testset assert ('user3', 'item0', trainset.global_mean) in testset