def benchmark_times(plays, iterations=3): times = defaultdict(lambda: defaultdict(list)) def store_time(model, name): def inner(iteration, elapsed): print(name, model.factors, iteration, elapsed) times[name][model.factors].append(elapsed) return inner output = defaultdict(list) for factors in range(32, 257, 32): for steps in [2, 3, 4]: model = AlternatingLeastSquares( factors=factors, use_native=True, use_cg=True, regularization=0, iterations=iterations, ) model.fit_callback = store_time(model, "cg%i" % steps) model.cg_steps = steps model.fit(plays) model = AlternatingLeastSquares(factors=factors, use_native=True, use_cg=False, regularization=0, iterations=iterations) model.fit_callback = store_time(model, "cholesky") model.fit(plays) if has_cuda: model = AlternatingLeastSquares( factors=factors, use_native=True, use_gpu=True, regularization=0, iterations=iterations, ) model.fit_callback = store_time(model, "gpu") model.fit(plays) # take the min time for the output output["factors"].append(factors) for name, stats in times.items(): output[name].append(min(stats[factors])) return output
def main(): args = get_args() set_seeds(args.seed) data_path = args.data_path data = read_data(root=data_path) result = evaluate(data=c.deepcopy(data), smoothing=args.smoothing, models_list=[ BM25Recommender(K=args.bm25_k), AlternatingLeastSquares(factors=args.als_factors, iterations=args.als_iters), ], r=args.rating, N=args.top_k) RATES = [5.5, 6] TOP_K = 100 predictions = mix_solutions(result=result, rates=RATES, pictures_num_to_leave=TOP_K) test_users = pd.DataFrame.from_dict(predictions).T.reset_index() test_users.rename({'index': 'user_id'}, inplace=True, axis=1) test_users.sort_values('user_id', inplace=True) test_users['predictions'] = test_users[list(range(TOP_K))].apply( lambda x: ' '.join(map(str, x)), axis=1) test_users[['user_id', 'predictions']].to_csv('sub.csv', index=False)
def test_cg_nan2(self): # test out Nan appearing in CG code (from https://github.com/benfred/implicit/issues/106) Ciu = random(m=100, n=100, density=0.0005, format='coo', dtype=np.float32, random_state=42, data_rvs=None).T.tocsr() configs = [{ 'use_native': True, 'use_gpu': False }, { 'use_native': False, 'use_gpu': False }] if HAS_CUDA: configs.append({'use_gpu': True}) for options in configs: model = AlternatingLeastSquares(factors=32, regularization=10, iterations=10, dtype=np.float32, **options) model.fit(Ciu, show_progress=False) self.assertTrue(np.isfinite(model.item_factors).all()) self.assertTrue(np.isfinite(model.user_factors).all())
def calculate_similar_event(path, output_filename): model = AlternatingLeastSquares() a, b = read_event_data(path) event, users = hfd5_from_dataframe(a, b, output_filename) users.eliminate_zeros() users.data = np.ones(len(users.data)) log.info("Start fitting") model.fit(users) user_count = np.ediff1d(users.indptr) to_generate = sorted(np.arange(len(event)), key=lambda x: -user_count[x]) with tqdm.tqdm(total=len(to_generate)) as progress: with codecs.open(output_filename, "w", "utf-8") as o: for eventid in to_generate: if users.indptr[eventid] != users.indptr[eventid + 1]: name = event[eventid] for other, score in model.similar_items( eventid, int(len(event) * 2 / 3) ): o.write(f"{name},{event[other]},{score}\n") progress.update(1)
def fit(self, train_df, col_user=cfg.USER_COL, col_item=cfg.ITEM_COL, col_rating=cfg.DEFAULT_RATING_COL, factors=100, confidence=5, regularization=0.1): """ Trains implicit ALS recommender on train data :param train_df: pandas DataFrame with train data :param col_user: str column name for user :param col_item: str column name for item :param col_rating: str column name for ratings :param factors: int number of factors to use in ALS model :param confidence: int as described in implicit documentation :param regularization: float higher values mean stronger regularization :return: None """ BaseRecommender.fit(self, train_df, col_user, col_item, col_rating) self.train_df[self.col_rating] = train_df[self.col_rating] * confidence self.uii_matrix = self.get_uii_matrix() self.als = AlternatingLeastSquares(factors=factors, use_gpu=False, regularization=regularization) self.als.fit(self.uii_matrix.T)
def train_and_evaluate(self): self.model = AlternatingLeastSquares(factors= 16, \ iterations = 100) print('model is going to fit!') self.model.fit(self.train.transpose()) print('model is already!') self.evaluate()
def calculate_similar_movies(output_filename, model_name="als", min_rating=4.0, variant="20m"): # read in the input data file start = time.time() titles, ratings = get_movielens(variant) # remove things < min_rating, and convert to implicit dataset # by considering ratings as a binary preference only ratings.data[ratings.data < min_rating] = 0 ratings.eliminate_zeros() ratings.data = np.ones(len(ratings.data)) log.info("read data file in %s", time.time() - start) # generate a recommender model based off the input params if model_name == "als": model = AlternatingLeastSquares() # lets weight these models by bm25weight. log.debug("weighting matrix by bm25_weight") ratings = (bm25_weight(ratings, B=0.9) * 5).tocsr() elif model_name == "bpr": model = BayesianPersonalizedRanking() elif model_name == "lmf": model = LogisticMatrixFactorization() elif model_name == "tfidf": model = TFIDFRecommender() elif model_name == "cosine": model = CosineRecommender() elif model_name == "bm25": model = BM25Recommender(B=0.2) else: raise NotImplementedError("TODO: model %s" % model_name) # train the model log.debug("training model %s", model_name) start = time.time() model.fit(ratings) log.debug("trained model '%s' in %s", model_name, time.time() - start) log.debug("calculating top movies") user_count = np.ediff1d(ratings.indptr) to_generate = sorted(np.arange(len(titles)), key=lambda x: -user_count[x]) log.debug("calculating similar movies") with tqdm.tqdm(total=len(to_generate)) as progress: with codecs.open(output_filename, "w", "utf8") as o: for movieid in to_generate: # if this movie has no ratings, skip over (for instance 'Graffiti Bridge' has # no ratings > 4 meaning we've filtered out all data for it. if ratings.indptr[movieid] != ratings.indptr[movieid + 1]: title = titles[movieid] for other, score in model.similar_items(movieid, 11): o.write("%s\t%s\t%s\n" % (title, titles[other], score)) progress.update(1)
def dump_factors(): numfactors = int(request.args['numfactors'].strip()) model = AlternatingLeastSquares(factors=numfactors, dtype=np.float32, use_gpu=False, iterations=30) model.approximate_recommend = False model.approximate_similar_items = False data = {'userid': [], 'productid': [], 'purchase_count': []} for userid in purchases: for productid in purchases[userid]: data['userid'].append(userid) data['productid'].append(productid) data['purchase_count'].append(purchases[userid][productid]) df = pd.DataFrame(data) df['userid'] = df['userid'].astype("category") df['productid'] = df['productid'].astype("category") userids = list(df['userid'].cat.categories) userids_reverse = dict(zip(userids, list(range(len(userids))))) productids = list(df['productid'].cat.categories) productids_reverse = dict(zip(productids, list(range(len(productids))))) purchases_matrix = coo_matrix((df['purchase_count'].astype(np.float32), (df['productid'].cat.codes.copy(), df['userid'].cat.codes.copy()))) print("Matrix shape: %s, max value: %.2f" % (np.shape(purchases_matrix), np.max(purchases_matrix))) purchases_matrix = bm25_weight(purchases_matrix, K1=2.0, B=0.25) purchases_matrix_T = purchases_matrix.T.tocsr() purchases_matrix = purchases_matrix.tocsr() # to support indexing in recommend/similar_items functions model.fit(purchases_matrix) np.savetxt('item_factors.csv', model.item_factors, delimiter=',') np.savetxt('user_factors.csv', model.user_factors, delimiter=',') with open('item_ids.csv', 'w') as f: for pid in productids_reverse: f.write("%s,%d,%s\n" % (pid, productids_reverse[pid], recommendation.sub(r',', ' ', productnames[pid]))) with open('user_ids.csv', 'w') as f: for uid in userids_reverse: f.write("%s,%d,%s\n" % (uid, userids_reverse[uid], recommendation.sub(r',', ' ', usernames[uid]))) return 'OK\n'
def __init__(self, user_df, song_df, k=100, knn_frac=0.5, max_overlap=0.2, cf_weighting_alpha=1, min_songs=5, mode='popular'): self.user_df = user_df self.song_df = song_df self.cf_weighting_alpha = cf_weighting_alpha self.knn_frac = knn_frac self.k = k self.max_overlap = max_overlap self.min_songs = min_songs self.mode = mode user_df_subset = user_df.loc[user_df['num_songs'] > (min_songs - 1)] self.kdtree = KDTree(user_df_subset['MUSIC'].tolist()) #build the collaborative filtering model with params hardcoded als_params = { 'factors': 16, 'dtype': np.float32, 'iterations': 2, 'calculate_training_loss': True } self.cf_model = AlternatingLeastSquares(**als_params)
def make_latent_feature(df: pd.DataFrame, index_col: str, value_col: str, n_factors: int, n_iterations: int, sum_col: Optional[str] = None): if sum_col is None: csr = make_count_csr(df, index_col=index_col, value_col=value_col) else: csr = make_sum_csr( df, index_col=index_col, value_col=value_col, col_to_sum=sum_col, ) model = AlternatingLeastSquares( factors=n_factors, dtype=np.float32, iterations=n_iterations, regularization=0.1, use_gpu=False, # True if n_factors >= 32 else False, ) np.random.seed(RANDOM_STATE) model.fit(csr.T) return model.user_factors
def __init__(self, *args, **kwargs): """ Construct an ALS recommender. The arguments are passed as-is to :py:class:`implicit.als.AlternatingLeastSquares`. """ from implicit.als import AlternatingLeastSquares super().__init__(AlternatingLeastSquares(*args, **kwargs))
def train_als(train_df, test_df, min_rating=4.0): # map each user/item to a unique numeric value train_df['user_id'] = train_df['user_id'].astype("category") train_df['item_id'] = train_df['item_id'].astype("category") ratings_csr = coo_matrix((train_df['rating'].astype(np.float32), (train_df['item_id'].cat.codes.copy(), train_df['user_id'].cat.codes.copy()))).tocsr() items = np.array(train_df['item_id'].cat.categories) users = np.array(train_df['user_id'].cat.categories) ratings = ratings_csr # remove things < min_rating, and convert to implicit dataset # by considering ratings as a binary preference only ratings.data[ratings.data < min_rating] = 0 ratings.eliminate_zeros() ratings.data = np.ones(len(ratings.data)) model = AlternatingLeastSquares() # lets weight these models by bm25weight. ratings = (bm25_weight(ratings, B=0.9) * 5).tocsr() # train the model start = time.time() model.fit(ratings) print("Training time: {}".format(time.time() - start)) return model, users, items, ratings
def test_cg_nan(self): # test issue with CG code that was causing NaN values in output: # https://github.com/benfred/implicit/issues/19#issuecomment-283164905 raw = [[0.0, 2.0, 1.5, 1.33333333, 1.25, 1.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 2.0, 1.5, 1.33333333, 1.25, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 2.0, 1.5, 1.33333333, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 2.0, 1.5, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.5, 1.33333333, 1.25, 1.2], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.5, 1.33333333, 1.25], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.5, 1.33333333], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0, 1.5], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 2.0], [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]] counts = csr_matrix(raw, dtype=np.float64) for use_native in (True, False): model = AlternatingLeastSquares(factors=3, regularization=0.01, dtype=np.float64, use_native=use_native, use_cg=True, use_gpu=False) model.fit(counts, show_progress=False) rows, cols = model.item_factors, model.user_factors self.assertFalse(np.isnan(np.sum(cols))) self.assertFalse(np.isnan(np.sum(rows)))
def test_factorize(self): counts = csr_matrix( [ [1, 1, 0, 1, 0, 0], [0, 1, 1, 1, 0, 0], [1, 0, 1, 0, 0, 0], [1, 1, 0, 0, 0, 0], [0, 0, 1, 1, 0, 1], [0, 1, 0, 0, 0, 1], [0, 0, 0, 0, 1, 1], ], dtype=np.float64, ) user_items = counts * 2 # try all 8 variants of native/python, cg/cholesky, and # 64 vs 32 bit factors options = [(dtype, cg, native, False) for dtype in (np.float32, np.float64) for cg in (False, True) for native in (False, True)] # also try out GPU support if available if HAS_CUDA: options.append((np.float32, False, False, True)) for dtype, use_cg, use_native, use_gpu in options: try: model = AlternatingLeastSquares( factors=6, regularization=0, dtype=dtype, use_native=use_native, use_cg=use_cg, use_gpu=use_gpu, random_state=42, ) model.fit(user_items, show_progress=False) rows, cols = model.item_factors, model.user_factors if use_gpu: rows, cols = rows.to_numpy(), cols.to_numpy() except Exception as e: self.fail(msg="failed to factorize matrix. Error=%s" " dtype=%s, cg=%s, native=%s gpu=%s" % (e, dtype, use_cg, use_native, use_gpu)) reconstructed = rows.dot(cols.T) for i in range(counts.shape[0]): for j in range(counts.shape[1]): self.assertAlmostEqual( counts[i, j], reconstructed[i, j], delta=0.0001, msg="failed to reconstruct row=%s, col=%s," " value=%.5f, dtype=%s, cg=%s, native=%s gpu=%s" % (i, j, reconstructed[i, j], dtype, use_cg, use_native, use_gpu), )
def benchmark_implicit(matrix, factors, reg, iterations): start = time.time() model = AlternatingLeastSquares(factors, regularization=reg, iterations=iterations, use_cg=True) model.fit(matrix) return time.time() - start
def _prep_for_fit(self, train_obs, **fit_params): # self.toggle_mkl_blas_1_thread(True) self._set_data(train_obs) self.set_params(**fit_params) self.model = AlternatingLeastSquares(**self.model_params) self.model.cg_steps = self.fit_params[ 'cg_steps'] # not passable to __init__() self._set_implib_train_mat(self.train_mat)
def fit(user_item_matrix, factors=20, regularization=0.001, iterations=15): """Обучает ALS""" model = AlternatingLeastSquares(factors=factors, regularization=regularization, iterations=iterations) model.fit(csr_matrix(user_item_matrix).T.tocsr()) return model
def get_baseline_cf_model(): als_params = { 'factors': 16, 'dtype': np.float32, 'iterations': 2, 'calculate_training_loss': True } cf_model = AlternatingLeastSquares(**als_params) return cf_model
def __init__(self, k, reg=1e-4, n_iters=15): """""" super().__init__() self.k = k self.reg = reg self.n_iters = n_iters self.als = AlternatingLeastSquares( k, regularization=reg, iterations=n_iters )
def calculate_similar_movies(input_path, output_filename, model_name="als", min_rating=4.0): # read in the input data file logging.debug("reading data from %s", input_path) start = time.time() ratings, movies, m = read_data(input_path, min_rating=min_rating) logging.debug("read data file in %s", time.time() - start) # generate a recommender model based off the input params if model_name == "als": model = AlternatingLeastSquares() # lets weight these models by bm25weight. logging.debug("weighting matrix by bm25_weight") m = bm25_weight(m, B=0.9) * 5 elif model_name == "bpr": model = BayesianPersonalizedRanking() elif model_name == "tfidf": model = TFIDFRecommender() elif model_name == "cosine": model = CosineRecommender() elif model_name == "bm25": model = BM25Recommender(B=0.2) else: raise NotImplementedError("TODO: model %s" % model_name) # train the model m = m.tocsr() logging.debug("training model %s", model_name) start = time.time() model.fit(m) logging.debug("trained model '%s' in %s", model_name, time.time() - start) logging.debug("calculating top movies") user_count = ratings.groupby('movieId').size() movie_lookup = dict( (i, m) for i, m in zip(movies['movieId'], movies['title'])) to_generate = sorted(list(movies['movieId']), key=lambda x: -user_count.get(x, 0)) with codecs.open(output_filename, "w", "utf8") as o: for movieid in to_generate: # if this movie has no ratings, skip over (for instance 'Graffiti Bridge' has # no ratings > 4 meaning we've filtered out all data for it. if m.indptr[movieid] == m.indptr[movieid + 1]: continue movie = movie_lookup[movieid] for other, score in model.similar_items(movieid, 11): o.write("%s\t%s\t%s\n" % (movie, movie_lookup[other], score))
def fit(self, n_factors=20, regularization=0.001, iterations=15, num_threads=4): """Обучает ALS""" self.model = AlternatingLeastSquares(factors=n_factors, regularization=regularization, iterations=iterations, num_threads=num_threads) self.model.fit(self.ds.csr_matrix) return self
def _add_als_recs(self, n_factors=20, regularization=0.001, iterations=20, num_threads=0): als_model = AlternatingLeastSquares(factors=n_factors, regularization=regularization, iterations=iterations, num_threads=num_threads) als_model.fit(csr_matrix(self.user_item_matrix).T.tocsr()) self.als_model = als_model als_recs = lambda i: [ self.id_to_itemid[rec[0]] for rec in als_model.recommend( userid=int(i), user_items=csr_matrix(self.user_item_matrix).tocsr(), N=self.first_model_rec_limit, filter_items=[self.itemid_to_id[999999]], recalculate_user=True, filter_already_liked_items=False) ] self.df_users['als_recommender'] = None self.df_users.loc[~self.df_users['id'].isnull(), 'als_recommender'] = self.df_users.loc[ ~self.df_users['id'].isnull(), 'id'].map(als_recs) self.df_users['als_recommender'] = self.df_users[ 'als_recommender'].map(lambda val: val if type(val) == type([]) else []) # adding embedings to df_users and df_items as features als_user_factors = pd.DataFrame( self.als_model.user_factors, columns=[ f'als_user_factor_{i}' for i in range(self.als_model.user_factors.shape[1]) ]) als_user_factors['id'] = als_user_factors.index self.df_users = pd.merge(left=self.df_users, right=als_user_factors, on='id', how='left') als_item_factors = pd.DataFrame( self.als_model.item_factors, columns=[ f'als_item_factor_{i}' for i in range(self.als_model.item_factors.shape[1]) ]) als_item_factors['id'] = als_item_factors.index self.df_items = pd.merge(left=self.df_items, right=als_item_factors, on='id', how='left')
def test_explain(self): counts = csr_matrix( [ [1, 1, 0, 1, 0, 0], [0, 1, 1, 1, 0, 0], [1, 4, 1, 0, 7, 0], [1, 1, 0, 0, 0, 0], [9, 0, 4, 1, 0, 1], [0, 1, 0, 0, 0, 1], [0, 0, 2, 0, 1, 1], ], dtype=np.float64, ) user_items = counts * 2 item_users = user_items.T model = AlternatingLeastSquares( factors=4, regularization=20, use_native=False, use_cg=False, use_gpu=False, iterations=100, random_state=23, ) model.fit(user_items, show_progress=False) userid = 0 # Assert recommendation is the the same if we recompute user vectors recs = model.recommend(userid, item_users, N=10) recalculated_recs = model.recommend(userid, item_users, N=10, recalculate_user=True) for (item1, score1), (item2, score2) in zip(recs, recalculated_recs): self.assertEqual(item1, item2) self.assertAlmostEqual(score1, score2, 4) # Assert explanation makes sense top_rec, score = recalculated_recs[0] score_explained, contributions, W = model.explain(userid, item_users, itemid=top_rec) scores = [s for _, s in contributions] items = [i for i, _ in contributions] self.assertAlmostEqual(score, score_explained, 4) self.assertAlmostEqual(score, sum(scores), 4) self.assertEqual(scores, sorted(scores, reverse=True), "Scores not in order") self.assertEqual([0, 2, 3, 4], sorted(items), "Items not seen by user") # Assert explanation with precomputed user weights is correct top_score_explained, top_contributions, W = model.explain( userid, item_users, itemid=top_rec, user_weights=W, N=2 ) top_scores = [s for _, s in top_contributions] top_items = [i for i, _ in top_contributions] self.assertEqual(2, len(top_contributions)) self.assertAlmostEqual(score, top_score_explained, 4) self.assertEqual(scores[:2], top_scores) self.assertEqual(items[:2], top_items)
def main(params): """Main function.""" # check for mandatory params if 'reference_repo' not in params: return {'error': 'Mandatory param reference_repo not present'} reference_repo = params['reference_repo'] LOGGER.info('reference_repo %s' % reference_repo) # get data LOGGER.info('read GBQ data') _GC_SVC_ACCOUNT['private_key_id'] = params['GC_SVC_PRIVATE_KEY_ID'] _GC_SVC_ACCOUNT['private_key'] = params['GC_SVC_PRIVATE_KEY'] data = pd.io.gbq.read_gbq(_QUERY, dialect="standard", project_id=_GC_SVC_ACCOUNT['project_id'], private_key=json.dumps(_GC_SVC_ACCOUNT)) # map each repo and user to a unique numeric value data['user'] = data['user'].astype("category") data['repo'] = data['repo'].astype("category") # dictionaries to translate names to ids and vice-versa repos = dict(enumerate(data['repo'].cat.categories)) repo_ids = {r: i for i, r in repos.items()} if reference_repo not in repo_ids: return {"message": "No result. Reference repo not in training set."} # create a sparse matrix of all the users/repos stars = coo_matrix( (np.ones(data.shape[0]), (data['repo'].cat.codes.copy(), data['user'].cat.codes.copy()))) # train model LOGGER.info('training model') model = AlternatingLeastSquares( factors=50, regularization=0.01, dtype=np.float64, # pylint: disable=no-member iterations=50) confidence = 40 model.fit(confidence * stars) similar_ids = model.similar_items(repo_ids[reference_repo]) LOGGER.info('found %d similar repos' % len(similar_ids)) similar_repos = [] for idx in range(1, len(similar_ids)): similar_repos.append(repos[similar_ids[idx][0]]) return { 'reference_repo': reference_repo, 'similar_repos': similar_repos, 'error': '' }
def fit(user_item_matrix): """Обучает ALS""" model = AlternatingLeastSquares(factors=100, regularization=0.01, iterations=15, num_threads=4) model.fit(csr_matrix(user_item_matrix).T.tocsr()) return model
def fit(user_item_matrix, n_factors=20, regularization=0.1, iterations=40, num_threads=0): """Обучает ALS""" model = AlternatingLeastSquares(factors=n_factors, regularization=regularization, iterations=iterations, num_threads=num_threads) model.fit(csr_matrix(user_item_matrix).T.tocsr()) return model
def _train_als(hyperparameters, train): h = hyperparameters model = AlternatingLeastSquares(factors=h['factors'], iterations=h['n_iter'], num_threads=nproc) model.fit(train) # test_eval = {'p@k': precision_at_k(model, train.T.tocsr(), factorization.T.tocsr(), K=10)} # val_eval = {'p@k': precision_at_k(model, train.T.tocsr(), validation.T.tocsr(), K=10)} return model
def fit(user_item_matrix, n_factors=20, regularization=0.001, iterations=15, num_threads=4, show_progress=False): """Обучает ALS""" model = AlternatingLeastSquares(factors=n_factors, regularization=regularization, iterations=iterations, num_threads=num_threads) model.fit(csr_matrix(user_item_matrix).T.tocsr(), show_progress=show_progress) return model
def load_recommender(als_model_file: str, index_file: str, item_feature_file: str = None, **kwargs) -> ImplicitRecommender: log.info("Loading als model") data = np.load(als_model_file, allow_pickle=True) model = AlternatingLeastSquares( factors=data['model.item_factors'].shape[1]) model.item_factors = data['model.item_factors'] model.YtY # This will initialize the _YtY instance variable which is used directly in internal methods if 'user_factors' in data: model.user_factors = data['model.user_factors'] user_labels = data['user_labels'] item_labels = data['item_labels'] if index_file is None: return ImplicitRecommender(model, user_labels, item_labels) elif index_file.endswith('.ann'): import annoy log.info("Loading annoy recommendation index") max_norm, extra = augment_inner_product_matrix(model.item_factors) recommend_index = annoy.AnnoyIndex(extra.shape[1], 'angular') recommend_index.load( index_file) # prefault=load_to_memory does not seem to work if item_feature_file is None: from .annoy import ImplicitAnnoyRecommender return ImplicitAnnoyRecommender(model, recommend_index, max_norm, user_labels, item_labels) else: log.info("Loading item features for recommendation") item_feature_data = pickle.load(open(item_feature_file, "rb")) tag_tfidf_transformer = item_feature_data['tag_tfidf_transformer'] tag_lookup = item_feature_data['tag_lookup'] item_embedding_weight = item_feature_data['item_embedding_weight'] from .annoy_item_features import ImplicitAnnoyItemFeatureRecommender return ImplicitAnnoyItemFeatureRecommender( model, recommend_index, max_norm, user_labels, item_labels, tag_tfidf_transformer, tag_lookup, item_embedding_weight) elif index_file.endswith('.hnsw'): import hnswlib from .hnsw import ImplicitHNSWRecommender log.info("Loading hnsw recommendation index") # we build the index in l2 space and load it in inner product space on purpose. # This space change gives us 0.96 recall l2_recommend_index = hnswlib.Index(space='ip', dim=model.item_factors.shape[1]) l2_recommend_index.load_index(index_file) l2_recommend_index.set_ef(kwargs.get('ef', 2000)) return ImplicitHNSWRecommender(model, l2_recommend_index, user_labels, item_labels) else: raise RecommenderException("Unsupported file type" + index_file)
def calculate_similar_movies(input_path, output_filename, model_name="als", min_rating=4.0): """ :param input_path: 训练数据集的路径 :param output_filename: 输出的文件名称 :param model_name: 采用的模型 :param min_rating: 过滤所需的阈值大小 :return: """ logging.debug("reading data from %s", input_path) start = time.time() rating_data, movies_data, m = read_data(input_path, min_rating=min_rating) logging.debug("reading data in %s", time.time() - start) if model_name == "als": model = AlternatingLeastSquares() logging.debug("weighting matrix by bm25_weight") m = bm25_weight(m, B=0.9) * 5 elif model_name == "tfidf": model = TFIDFRecommender() elif model_name == "cosine": model = CosineRecommender() elif model_name == "bm25": model = BM25Recommender() else: raise NotImplementedError("TODU: model %s" % model_name) m = m.tocsr() logging.debug("Training model :%s" % model_name) start = time.time() model.fit(m) logging.debug("trained model '%s' in %s", model_name, time.time() - start) logging.debug("calculating top movies") user_count = rating_data.groupby("movieId").size() movie_lookup = dict((i, m) for i,m in zip(movies_data['movieId'], movies_data['title'])) to_generate = sorted(list(movies_data['movieId']), key=lambda x: -user_count.get(x, 0)) with open(output_filename, "w") as o: for movieid in to_generate: if(m.indptr[movieid] == m.indptr[movieid + 1]): continue movie = movie_lookup[movieid] for other, score in model.similar_items(movieid, 11): o.write("%s\t%s\t%s\n" % (movie, movie_lookup[other], score))