def test_ii_batch_accuracy(): from lenskit.algorithms import basic from lenskit.algorithms import bias import lenskit.crossfold as xf from lenskit import batch import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings ii_algo = knn.ItemItem(30) algo = basic.Fallback(ii_algo, bias.Bias()) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) return batch.predict(algo, test, n_jobs=4) preds = pd.concat( (eval(train, test) for (train, test) in xf.partition_users(ratings, 5, xf.SampleFrac(0.2)))) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.70, abs=0.025) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(0.90, abs=0.05)
def test_batch_rmse(): import lenskit.crossfold as xf import lenskit.batch as batch import lenskit.algorithms.basic as bl ratings = lktu.ml100k.ratings algo = bl.Bias(damping=5) def eval(train, test): algo.fit(train) preds = batch.predict(algo, test) return preds.set_index(['user', 'item']) results = pd.concat( (eval(train, test) for (train, test) in xf.partition_users(ratings, 5, xf.SampleN(5)))) user_rmse = results.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) # we should have all users users = ratings.user.unique() assert len(user_rmse) == len(users) missing = np.setdiff1d(users, user_rmse.index) assert len(missing) == 0 # we should not have any missing values assert all(user_rmse.notna()) # we should have a reasonable mean assert user_rmse.mean() == approx(0.93, abs=0.05)
def test_tf_bmf_batch_accuracy(tf_session): from lenskit.algorithms import basic import lenskit.crossfold as xf from lenskit import batch import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings algo = lktf.BiasedMF(25, damping=10, batch_size=1024, epochs=20, rng_spec=42) algo = basic.Fallback(algo, basic.Bias(damping=10)) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) return batch.predict(algo, test) folds = xf.partition_users(ratings, 5, xf.SampleFrac(0.2)) preds = pd.concat(eval(train, test) for (train, test) in folds) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.83, abs=0.025) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(1.03, abs=0.05)
def test_fsvd_batch_accuracy(): from lenskit.algorithms import basic import lenskit.crossfold as xf from lenskit import batch import lenskit.metrics.predict as pm ratings = lktu.ml100k.load_ratings() svd_algo = svd.FunkSVD(25, 125, damping=10) algo = basic.Fallback(svd_algo, basic.Bias(damping=10)) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) return batch.predict(algo, test) folds = xf.partition_users(ratings, 5, xf.SampleFrac(0.2)) preds = pd.concat(eval(train, test) for (train, test) in folds) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.74, abs=0.025) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(0.92, abs=0.05)
def test_bias_batch_predict(ncpus): from lenskit.algorithms import basic import lenskit.crossfold as xf from lenskit import batch import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings algo = basic.Bias(damping=5) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) recs = batch.predict(algo, test, n_jobs=ncpus) return recs preds = pd.concat((eval(train, test) for (train, test) in xf.partition_users(ratings, 5, xf.SampleFrac(0.2)))) _log.info('analyzing predictions') rmse = pm.rmse(preds.prediction, preds.rating) _log.info('RMSE is %f', rmse) assert rmse == pytest.approx(0.95, abs=0.1)
def eval(algo, train, test): fittable = util.clone(algo) algo.fit(train) users = test.user.unique() preds = algo.predict(test) rmse = predict.rmse(preds, test['rating']) return rmse
def test_als_batch_accuracy(): from lenskit.algorithms import basic import lenskit.crossfold as xf import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings lu_algo = als.BiasedMF(25, iterations=20, damping=5, method='lu') cd_algo = als.BiasedMF(25, iterations=25, damping=5, method='cd') # algo = basic.Fallback(svd_algo, basic.Bias(damping=5)) def eval(train, test): _log.info('training LU') lu_algo.fit(train) _log.info('training CD') cd_algo.fit(train) _log.info('testing %d users', test.user.nunique()) return test.assign(lu_pred=lu_algo.predict(test), cd_pred=cd_algo.predict(test)) folds = xf.partition_users(ratings, 5, xf.SampleFrac(0.2)) preds = pd.concat(eval(train, test) for (train, test) in folds) preds['abs_diff'] = np.abs(preds.lu_pred - preds.cd_pred) _log.info('predictions:\n%s', preds.sort_values('abs_diff', ascending=False)) _log.info('diff summary:\n%s', preds.abs_diff.describe()) lu_mae = pm.mae(preds.lu_pred, preds.rating) assert lu_mae == approx(0.73, abs=0.03) cd_mae = pm.mae(preds.cd_pred, preds.rating) assert cd_mae == approx(0.73, abs=0.03) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.lu_pred, df.rating)) assert user_rmse.mean() == approx(0.91, abs=0.05) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.cd_pred, df.rating)) assert user_rmse.mean() == approx(0.91, abs=0.05)
def test_tf_isvd(ml20m): algo = lenskit_tf.IntegratedBiasMF(20) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) return batch.predict(algo, test) folds = xf.sample_users(ml20m, 2, 5000, xf.SampleFrac(0.2)) preds = pd.concat(eval(train, test) for (train, test) in folds) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.60, abs=0.025) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(0.92, abs=0.05)
def test_global_metric(): import lenskit.crossfold as xf import lenskit.batch as batch from lenskit.algorithms.bias import Bias train, test = next( xf.sample_users(lktu.ml_test.ratings, 1, 200, xf.SampleFrac(0.5))) algo = Bias() algo.fit(train) preds = batch.predict(algo, test) rmse = pm.global_metric(preds) assert rmse == pm.rmse(preds.prediction, preds.rating) mae = pm.global_metric(preds, metric=pm.mae) assert mae == pm.mae(preds.prediction, preds.rating)
def test_uu_batch_accuracy(): from lenskit.algorithms import basic import lenskit.crossfold as xf import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings uu_algo = knn.UserUser(30) algo = basic.Fallback(uu_algo, basic.Bias()) folds = xf.partition_users(ratings, 5, xf.SampleFrac(0.2)) preds = [__batch_eval((algo, train, test)) for (train, test) in folds] preds = pd.concat(preds) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.71, abs=0.028) user_rmse = preds.groupby('user').apply(lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(0.91, abs=0.055)
def test_user_metric(): import lenskit.crossfold as xf import lenskit.batch as batch from lenskit.algorithms.bias import Bias train, test = next( xf.sample_users(lktu.ml_test.ratings, 1, 200, xf.SampleFrac(0.5))) algo = Bias() algo.fit(train) preds = batch.predict(algo, test) rmse = pm.user_metric(preds) u_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert rmse == approx(u_rmse.mean()) mae = pm.user_metric(preds, metric=pm.mae) u_mae = preds.groupby('user').apply( lambda df: pm.mae(df.prediction, df.rating)) assert mae == approx(u_mae.mean())
def test(self, path): algo_pop = Bias() algo_als5 = als.BiasedMF(5) def eval(aname, algo, train, test, all_preds): fittable = util.clone(algo) fittable = Recommender.adapt(fittable) fittable.fit(train) # predict ratings preds = batch.predict(fittable, test) preds['Algorithm'] = aname all_preds.append(preds) if '100k' in path: ml100k = ML100K(path) ratings = ml100k.ratings elif '1m' in path: ml100k = ML1M(path) ratings = ml100k.ratings elif '10m' in path: ml100k = ML10M(path) ratings = ml100k.ratings else: mlsmall = MovieLens(path) ratings = mlsmall.ratings print(ratings.head()) all_preds = [] test_data = [] for train, test in xf.partition_users( ratings[['user', 'item', 'rating']], 5, xf.SampleFrac(0.2)): test_data.append(test) eval('MF', algo_als5, train, test, all_preds) preds = pd.concat(all_preds, ignore_index=True) preds_mf = preds[preds['Algorithm'].str.match('MF')] test_data = pd.concat(test_data, ignore_index=True) print('RMSE MF:', rmse(preds_mf['prediction'], preds_mf['rating']))
def test_als_batch_accuracy(): from lenskit.algorithms import basic import lenskit.crossfold as xf import lenskit.metrics.predict as pm ratings = lktu.ml100k.ratings svd_algo = als.BiasedMF(25, iterations=20, damping=5) algo = basic.Fallback(svd_algo, basic.Bias(damping=5)) def eval(train, test): _log.info('running training') algo.fit(train) _log.info('testing %d users', test.user.nunique()) return test.assign(prediction=algo.predict(test)) folds = xf.partition_users(ratings, 5, xf.SampleFrac(0.2)) preds = pd.concat(eval(train, test) for (train, test) in folds) mae = pm.mae(preds.prediction, preds.rating) assert mae == approx(0.73, abs=0.025) user_rmse = preds.groupby('user').apply( lambda df: pm.rmse(df.prediction, df.rating)) assert user_rmse.mean() == approx(0.91, abs=0.05)
def test_rmse_series_missing_value_ignore(): rmse = pm.rmse(pd.Series([1, 3], ['a', 'd']), pd.Series([3, 4, 1], ['a', 'b', 'c']), missing='ignore') assert rmse == approx(2)
def test_rmse_series_missing_value_error(): with raises(ValueError): pm.rmse(pd.Series([1, 3], ['a', 'd']), pd.Series([3, 4, 1], ['a', 'b', 'c']))
def test_rmse_series_subset_axis(): rmse = pm.rmse(pd.Series([1, 3], ['a', 'c']), pd.Series([3, 4, 1], ['a', 'b', 'c'])) assert rmse == approx(2)
test_bool = True train = pd.read_pickle("../data/ml-1m-split/train.pkl") val = pd.read_pickle("../data/ml-1m-split/val.pkl") test = pd.read_pickle("../data/ml-1m-split/test.pkl") num_factors = 30 num_iters = 100 model = BiasedMF(num_factors, iterations=num_iters) print("Fitting model...") model.fit(train) print("Making validation predictions...") val_preds = predict(model, val) val_result = rmse(val_preds["prediction"], val_preds["rating"]) if test_bool: print("Making test predictions...") test_preds = predict(model, test) test_result = rmse(test_preds["prediction"], test_preds["rating"]) else: test_result = 0 print("============= RESULTS =============\nFactors: {}\nIterations: {}\nValidation RMSE: {}\nTest RMSE: {}" \ .format(num_factors, num_iters, val_result, test_result))
algo_pop = Bias() algo_ii = knn.ItemItem(20) def eval(aname, algo, train, test, all_preds): fittable = util.clone(algo) fittable = Recommender.adapt(fittable) fittable.fit(train) preds = batch.predict(fittable, test) preds['Algorithm'] = aname all_preds.append(preds) all_preds = [] test_data = [] for train, test in xf.partition_users(df_ratings, 5, xf.SampleFrac(0.2)): test_data.append(test) eval('BIAS', algo_pop, train, test, all_preds) eval('II', algo_ii, train, test, all_preds) preds = pd.concat(all_preds, ignore_index=True) preds_ii = preds[preds['Algorithm'].str.match('BIAS')] print(preds_ii.head()) preds_bias = preds[preds['Algorithm'].str.match('BIAS')] print(preds_bias.head()) test_data = pd.concat(test_data, ignore_index=True) print('RMSE BIAS: ', rmse(preds_bias['prediction'], preds_bias['rating'])) print('RMSE II: ', rmse(preds_ii['prediction'], preds_ii['rating']))
set_matplotlib_formats('svg') results.groupby('Algorithm').ndcg.mean().plot.bar() # %% results.groupby('Algorithm').precision.mean() # %% results.groupby('Algorithm').recall.mean() # %% [markdown] # ### 2. Prediction Metrics: RMSE # %% user_rmse = (preds.groupby( ['Algorithm', 'user']).apply(lambda df: rmse(df.prediction, df.rating))).dropna() print(user_rmse.groupby('Algorithm').mean()) print(user_rmse['domex']) plt.boxplot((user_rmse['ii'], user_rmse['trst'], user_rmse['socsim'], user_rmse['domex'], user_rmse['hierch'], user_rmse['socap'], user_rmse['soxsim'], user_rmse['symp'], user_rmse['rel']), labels=[ 'ii', 'trst', 'socsim', 'domex', 'hierch', 'socap', 'soxsim', 'symp', 'rel' ]) plt.show() """ print('Item Item Knn: max rmse per user = '******', min rmse per user = '******', average rmse per user = '******'Trust: max rmse per user = '******', min rmse per user = '******', average rmse per user = ' + str(user_rmse_trst.mean()))