def test_performances(): """Test the returned dict. Also do dumping.""" current_dir = os.path.dirname(os.path.realpath(__file__)) folds_files = [(current_dir + '/custom_train', current_dir + '/custom_test')] reader = Reader(line_format='user item rating', sep=' ', skip_lines=3, rating_scale=(1, 5)) data = Dataset.load_from_folds(folds_files=folds_files, reader=reader) algo = NormalPredictor() tmp_dir = tempfile.mkdtemp() # create tmp dir performances = evaluate(algo, data, measures=['RmSe', 'Mae'], with_dump=True, dump_dir=tmp_dir, verbose=2) shutil.rmtree(tmp_dir) # remove tmp dir print(performances) assert performances['RMSE'] is performances['rmse'] assert performances['MaE'] is performances['mae']
def test_unknown_user_or_item(): """Ensure that all algorithms act gracefully when asked to predict a rating of an unknown user, an unknown item, and when both are unknown. """ reader = Reader(line_format='user item rating', sep=' ', skip_lines=3, rating_scale=(1, 5)) file_path = os.path.dirname(os.path.realpath(__file__)) + '/custom_train' data = Dataset.load_from_file(file_path=file_path, reader=reader) for trainset, testset in data.folds(): pass # just need trainset and testset to be set klasses = (NormalPredictor, BaselineOnly, KNNBasic, KNNWithMeans, KNNBaseline, SVD, SVDpp) for klass in klasses: algo = klass() algo.train(trainset) algo.predict(0, 'unknown_item') algo.predict('unkown_user', 0) algo.predict('unkown_user', 'unknown_item')
def get_my_recs(self, my_ratings_dict): self.my_rated_movies = list(my_ratings_dict.keys()) personal_dict = {'movieId' : list(my_ratings_dict.keys()), 'rating' : list(my_ratings_dict.values())} personal_df = pd.DataFrame(personal_dict) id_to_title = {mid: m for m, mid in zip(self.movies_df['title'].values, self.movies_df['movieId'])} # arbitrarily calling ourselves userId #10000 personal_df['userId'] = [10000 for _ in range(len(personal_df))] personal_df = scale_ratings(personal_df) new_df = pd.concat([self.ratings_df, personal_df], ignore_index=True) # new_df_scaled = scale_ratings(new_df) scaled_reader = Reader(rating_scale=(0, 1)) new_data = convert_df_to_data(new_df, scaled_reader) new_data.split(n_folds=5) new_trainset = new_data.build_full_trainset() self.model.train(new_trainset) unique_movie_ids = self.ratings_df['movieId'].unique() my_predictions = [self.model.predict(10000, mid) for mid in unique_movie_ids if id_to_title[mid] not in self.my_rated_movies] my_predictions = sorted(my_predictions, key=lambda pred : pred.est, reverse=True) my_estimates = [(id_to_title[pred.iid], pred.est) for pred in my_predictions] self.my_estimates = my_estimates
def get_your_recs(personal_df, df, movies_df, model): # arbitrarily calling ourselves userId #700 personal_df['userId'] = [700 for _ in range(len(personal_df))] new_df = pd.concat([df, personal_df], ignore_index=True) new_df_scaled = scale_ratings(new_df) scaled_reader = Reader(rating_scale=(0, 1)) new_data = convert_df_to_data(new_df_scaled, scaled_reader) new_data.split(n_folds=5) new_trainset = new_data.build_full_trainset() model.train(new_trainset) unique_movie_ids = df['movieId'].unique() my_predictions = [model.predict(700, mid) for mid in unique_movie_ids] my_highest_est = np.array([pred.est for pred in my_predictions]).max() my_top_recs_ids = [pred.iid for pred in my_predictions if pred.est >= my_highest_est] id_to_title = {mid: m for m, mid in zip(movies_df['title'].values, movies_df['movieId'])} # id_to_title = {mid : title for title, mid in movies_id_dict.items()} my_top_recs = [id_to_title[mid] for mid in my_top_recs_ids] my_predictions = sorted(my_predictions, key=lambda pred : pred.est, reverse=True) all_my_estimates = [(id_to_title[pred.iid], pred.est) for pred in my_predictions] all_my_estimates_2016 = [est for est in all_my_estimates if '(2016)' in est[0]] return my_top_recs, all_my_estimates_2016, all_my_estimates
def load_from_panda(df): """ Transform pandas dataframe into surprise dataframe Args: df: pandas dataframe to transform Returns: data: surprise dataframe """ #['userId', 'movieId', 'rating', 'timestamp'] reader = Reader(rating_scale=(1, 5)) data = Dataset.load_from_df(df, reader) return data
def test_knns(): """Ensure the k and min_k parameters are effective for knn algorithms.""" # the test and train files are from the ml-100k dataset (10% of u1.base and # 10 % of u1.test) train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) # Actually, as KNNWithMeans and KNNBaseline have back up solutions for when # there are not enough neighbors, we can't really test them... klasses = (KNNBasic, ) # KNNWithMeans, KNNBaseline) k, min_k = 20, 5 for klass in klasses: algo = klass(k=k, min_k=min_k) for trainset, testset in data.folds(): algo.train(trainset) predictions = algo.test(testset) for pred in predictions: if not pred.details['was_impossible']: assert min_k <= pred.details['actual_k'] <= k
# First map the predictions to each user. top_n = defaultdict(list) for uid, iid, true_r, est, _ in predictions: top_n[uid].append((iid, est)) # Then sort the predictions for each user and retrieve the k highest ones. for uid, user_ratings in top_n.items(): user_ratings.sort(key=lambda x: x[1], reverse=True) top_n[uid] = user_ratings[:n] return top_n # First train an SVD algorithm on the movielens dataset. file_path = 'ratings_robotics.dat' reader = Reader(line_format='user item rating', rating_scale=(1, 5), sep=' ') data = Dataset.load_from_file(file_path, reader=reader) #data = Dataset.load_builtin('ml-100k') trainset = data.build_full_trainset() algo = SVD() algo.train(trainset) # Than predict ratings for all pairs (u, i) that are NOT in the training set. testset = trainset.build_anti_testset() predictions = algo.test(testset) top_n = get_top_n(predictions, n=10) r = redis.Redis('localhost') r.hset('test', 'suggestions_dict',json.dumps(top_n)) # Print the recommended items for each user
from __future__ import (absolute_import, division, print_function, unicode_literals) import os from surprise.prediction_algorithms import SVD from surprise.prediction_algorithms import SVDpp from surprise.dataset import Dataset from surprise.dataset import Reader from surprise.evaluate import evaluate # the test and train files are from the ml-100k dataset (10% of u1.base and # 10 % of u1.test) train_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_train') test_file = os.path.join(os.path.dirname(__file__), './u1_ml100k_test') data = Dataset.load_from_folds([(train_file, test_file)], Reader('ml-100k')) def test_SVD_parameters(): """Ensure that all parameters are taken into account.""" # The baseline against which to compare. algo = SVD(n_factors=1, n_epochs=1) rmse_default = evaluate(algo, data, measures=['rmse'])['rmse'] # n_factors algo = SVD(n_factors=2, n_epochs=1) rmse_factors = evaluate(algo, data, measures=['rmse'])['rmse'] assert rmse_default != rmse_factors # n_epochs
from sklearn.metrics import jaccard_similarity_score as jaccard # data preparation for collaborative filtering files=glob('homework/*.txt') users=pd.read_csv(files[0], sep='\t', header=0) hotels=pd.read_csv(files[1], sep='\t', header=0) activity=pd.read_csv(files[2], sep='\t', header=0).drop(16135).reset_index().drop('index', axis=1) # remove the outlier activity_count=activity.assign(browse=1).groupby(['user', 'hotel']).count().reset_index() data=users.assign(key=1).merge(hotels.assign(key=1), on='key', how='inner').drop('key', axis=1) data=data.merge(activity_count, on=['user', 'hotel'], how='left') data['browse']=data.browse.fillna(0) data=data[['user', 'hotel', 'browse']] # tentatively CV test for some algorithms reader = Reader(rating_scale=(0, 1)) data = Dataset.load_from_df(data, reader) data_cv=data data_cv.split(n_folds=5) # SVD test svd = SVD() perf = evaluate(svd, data, measures=['RMSE']) print_perf(perf) # MSE 0.052 param_svd = {'n_factors': [50, 100], 'lr_all': [0.003, 0.005], 'reg_all': [0.05, 0.1, 0.5]} gs = GridSearch(SVD, param_svd, measures=['RMSE']) gs.evaluate(data_cv) # RMSE 0.2272 ~ 0.2284, after many tests notice 0.2272 is a benchmark, 100, 0.003, 0.1
self.model.train(new_trainset) unique_movie_ids = self.ratings_df['movieId'].unique() my_predictions = [self.model.predict(10000, mid) for mid in unique_movie_ids if id_to_title[mid] not in self.my_rated_movies] my_predictions = sorted(my_predictions, key=lambda pred : pred.est, reverse=True) my_estimates = [(id_to_title[pred.iid], pred.est) for pred in my_predictions] self.my_estimates = my_estimates if __name__ == '__main__': movies_df = pd.read_csv('data/movies/movies.csv') # data = Dataset.load_builtin('ml-100k') # data.split(n_folds=5) scaled_reader = Reader(rating_scale=(0, 1)) reader = Reader(rating_scale=(1, 5)) df = pd.read_csv('data/movies/ratings.csv') scaled_df = scale_ratings(df) scaled_data = convert_df_to_data(scaled_df, scaled_reader) scaled_data.split(n_folds=5) data = convert_df_to_data(df, reader) data.split(n_folds=5) # plot some EDA figures: plot_average_rating_hist(df) # Cross Valdiation Tests for different Classification Models: models = [] models.append(('GM', GlobalMean()))
from surprise.dataset import Reader filePlace = "C:\\Users\\22560\\PycharmProjects\\lastFM\\networkData\\" gc.collect() # read train data os.chdir("C:\\Users\\22560\\PycharmProjects\\lastFM\\hetrec2011-lastfm-2k") train = pd.read_csv("trainAfterReg.csv") algo = SVD() reader = Reader(rating_scale=(train.weight.min(),train.weight.max())) data = Dataset.load_from_df(train[['userID', 'artistID', 'weight']], reader) data.split(3) perf = evaluate(algo, data, measures=['RMSE', 'MAE']) print_perf(perf)