def main(): movielens = fetch_movielens() train = movielens['train'] test = movielens['test'] print(train.shape) print(test.shape) model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=5) k = 10 train_recall = recall_at_k(model, train, k=k).mean() test_recall = recall_at_k(model, test, k=k).mean() print(f'recall_at_{k}(train): {train_recall}') print(f'recall_at_{k}(test) : {test_recall}') train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print(f'auc_score(train): {train_auc}') print(f'auc_score(test) : {test_auc}') y_train_preds = model.predict_rank(train) y_test_preds = model.predict_rank(test) train_dcg = dcg_score(train.toarray(), y_train_preds.toarray()) test_dcg = dcg_score(test.toarray(), y_test_preds.toarray()) print(f'dcg_score(train): {train_dcg}') print(f'dcg_score(test) : {test_dcg}') print('DONE') return 0
def evaluate(self, model, user_items_train): print("Splitting the data into train/test set...\n") train, test = cross_validation.random_train_test_split( user_items_train) print(train, test) print("Evaluating methods...\n") train_recall_10 = recall_at_k(model, train, k=10).mean() test_recall_10 = recall_at_k(model, test, k=10).mean() train_recall_20 = recall_at_k(model, train, k=20).mean() test_recall_20 = recall_at_k(model, test, k=20).mean() train_precision_10 = precision_at_k(model, train, k=10).mean() test_precision_10 = precision_at_k(model, test, k=10).mean() train_precision_20 = precision_at_k(model, train, k=20).mean() test_precision_20 = precision_at_k(model, test, k=20).mean() print("Train : Recall@10:{0:.3f}, Recall@20:{1:.3f}".format( train_recall_10, train_recall_20)) print("Test : Recall@10:{0:.3f}, Recall@20:{1:.3f}".format( test_recall_10, test_recall_20)) print("Train: Precision@10:{0:.3f}, Precision@20:{1:.3f}".format( train_precision_10, train_precision_20)) print("Test: Precision@10:{0:.3f}, Precision@20:{1:.3f}".format( test_precision_10, test_precision_20))
def test(train_matrix_path, test_matrix_path): train_rating_df = pd.read_csv(train_matrix_path, sep=',', names=['profile', 'item', 'rating']) train_row, train_col, train_ratings = train_rating_df['profile'].values, train_rating_df['item'].values, \ train_rating_df['rating'].values n_user = 627 #np.max(train_row) + 1 n_item = 12 #np.max(train_col) + 1 train_data = csr_matrix((train_ratings, (train_row, train_col)), shape=(n_user, n_item)) model = LightFM(loss='warp', no_components=200, item_alpha=0.001, user_alpha=0.001) model.fit(train_data, epochs=20, num_threads=30) test_rating_df = pd.read_csv(test_matrix_path, sep=',', names=['profile', 'item', 'rating']) test_row, test_col, test_ratings = test_rating_df['profile'].values, test_rating_df['item'].values, \ test_rating_df['rating'].values test_data = csr_matrix((test_ratings, (test_row, test_col)), shape=(n_user, n_item)) print("Train precision: %.5f" % recall_at_k(model, train_data, k=6, num_threads=1).mean()) print("Test precision: %.5f" % recall_at_k(model, test_data, k=6, num_threads=1).mean())
def test_recall_at_k(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss="bpr") model.fit_partial(test) for k in (10, 5, 1): # Without omitting train interactions recall = evaluation.recall_at_k(model, test, k=k) expected_mean_recall = _recall_at_k(model, test, k) assert np.allclose(recall.mean(), expected_mean_recall) assert len(recall) == (test.getnnz(axis=1) > 0).sum() assert ( len(evaluation.recall_at_k(model, train, preserve_rows=True)) == test.shape[0] ) # With omitting train interactions recall = evaluation.recall_at_k(model, test, k=k, train_interactions=train) expected_mean_recall = _recall_at_k(model, test, k, train=train) assert np.allclose(recall.mean(), expected_mean_recall)
def test_recall_at_k(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss="bpr") model.fit_partial(test) for k in (10, 5, 1): # Without omitting train interactions recall = evaluation.recall_at_k(model, test, k=k) expected_mean_recall = _recall_at_k(model, test, k) assert np.allclose(recall.mean(), expected_mean_recall) assert len(recall) == (test.getnnz(axis=1) > 0).sum() assert (len(evaluation.recall_at_k( model, train, preserve_rows=True)) == test.shape[0]) # With omitting train interactions recall = evaluation.recall_at_k(model, test, k=k, train_interactions=train) expected_mean_recall = _recall_at_k(model, test, k, train=train) assert np.allclose(recall.mean(), expected_mean_recall)
def collab_filtering(): """ implements collaborative filtering version by using only the rating data from movielens dataset :return: """ data = fetch_movielens() for key, value in data.items(): print(key, type(value), value.shape) train = data['train'] test = data['test'] print( 'The dataset has %s users and %s items, ' 'with %s interactions in the test and %s interactions in the training set.' % (train.shape[0], train.shape[1], test.getnnz(), train.getnnz())) model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=50, num_threads=5) train_precision = precision_at_k(model, train, k=10).mean() test_precision = precision_at_k(model, test, k=10).mean() train_recall = recall_at_k(model, test, k=10).mean() test_recall = recall_at_k(model, test, k=10).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) print('Recall: train %.2f, test %.2f.' % (train_recall, test_recall)) print('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) model = LightFM(learning_rate=0.05, loss='warp') #resume training from the model's previous state model.fit_partial(train, epochs=50, num_threads=5) train_precision = precision_at_k(model, train, k=10).mean() test_precision = precision_at_k(model, test, k=10).mean() train_recall = recall_at_k(model, test, k=10).mean() test_recall = recall_at_k(model, test, k=10).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print("*****************") print("After re-training") print('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) print('Recall: train %.2f, test %.2f.' % (train_recall, test_recall)) print('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) #check sample recommendation sample_recommendation(model, data, [3, 25, 450])
def train (self, interactions, test_percentage=0.25, n_components=30, learning_rate = 0.5, loss='warp', model_k=15, n_jobs = 4, epoch=30, evaluate_k = 50): from lightfm.evaluation import precision_at_k from lightfm.evaluation import recall_at_k from lightfm.cross_validation import random_train_test_split train, test = random_train_test_split(interactions, test_percentage=test_percentage, random_state=None) mf_model = self.runMF(interactions = train, n_components = n_components, learning_rate = learning_rate, loss = loss, k = model_k, epoch = epoch, n_jobs = n_jobs) precise = precision_at_k(mf_model, test_interactions = test, k = evaluate_k) recall = recall_at_k(mf_model, test_interactions = test, k = evaluate_k) precise_test = precise.mean() recall_test = recall.mean() return mf_model, precise_test, recall_test
def run_BPR(split, user_item, prec_rec_at): # train_sparse = scipy.sparse.csr_matrix(train_splt.values) # test_sparse = scipy.sparse.csr_matrix(test_split.values) train = user_item test = user_item * 0 for index, row in split.iterrows(): test['u' + str(row['userId'])]['m' + str(row['movieId'])] = row['rating'] train['u' + str(row['userId'])]['m' + str(row['movieId'])] = 0 train_sparse = scipy.sparse.csr_matrix(train.values) test_sparse = scipy.sparse.csr_matrix(test.values) model = LightFM(loss='bpr') model.fit(train_sparse, epochs=30, num_threads=4) # print("####################################################################################") # print("Test precision: %.2f" % precision_at_k(model, test_sparse, k=prec_rec_at).mean()) # print("Test recall: %.2f" % recall_at_k(model, test_sparse, k=prec_rec_at).mean()) # print("####################################################################################") precision = precision_at_k(model, test_sparse, k=prec_rec_at).mean() recall = recall_at_k(model, test_sparse, k=prec_rec_at).mean() return precision, recall
def test_LightFM_model(model, test_interactions, train_interactions, user_features, movie_features, k=5): test_precision = precision_at_k(model, test_interactions, train_interactions, k=k, user_features=user_features, item_features=movie_features, num_threads=2).mean() test_recall = recall_at_k(model, test_interactions, train_interactions, k=k, user_features=user_features, item_features=movie_features, num_threads=2).mean() test_auc = auc_score(model, test_interactions, train_interactions, user_features=user_features, item_features=movie_features, num_threads=2).mean() print('Model') print('Precision at k=', str(k), ': ', round(test_precision, 3), sep='') print('Recall at k=', str(k) + ': ', round(test_recall, 3), sep='') print('AUC: ', round(test_auc, 3), sep='') return ({ 'precision': round(test_precision, 3), 'recall': round(test_recall, 3), 'auc': round(test_auc, 3) })
def evaluate_model(model, metric, test, train): """ Evaluate trained model on the test set, using one of the three available accuracy metrics AUC: the probability that a randomly chosen positive example has a higher score than a randomly chosen negative example. Precision: the fraction of known positives in the first k positions of the ranked list of results. Recall: the number of positive items in the first k positions of the ranked list of results divided by the number of positive items in the test period. :param model:(LightFM, required) - model to be evaluated :param metric:(string, required) - accuracy metric to be used, one of ['auc', 'precision', 'recall'] :param test:(COO matrix, required) - known positives used to test the model :param train:(COO matrix, required) - training set; these interactions will be omitted from the score calculations to avoid re-recommending known positives. :return: test_score (float) - score computed on the test set """ try: # make sure the metric is correct assert metric in ['auc', 'precision', 'recall'] if metric == 'auc': test_score = auc_score(model, test, train).mean() elif metric == 'precision': test_score = precision_at_k(model, test, train, k=5).mean() else: test_score = recall_at_k(model, test, train, k=5).mean() return test_score except AssertionError: print('The metric provided is not correct or available!')
def best_reccomendation(): #define variables best = 0.0 best_model = '' for model in models: score = 0.0 pak_score = evaluation.precision_at_k(model, data2['test']) score += np.mean(pak_score) rak_score = evaluation.recall_at_k(model, data2['test']) score += np.mean(rak_score) auc_score = evaluation.auc_score(model, data2['test']) score += np.mean(auc_score) rr_score = evaluation.reciprocal_rank(model, data2['test']) score += np.mean(rr_score) print(score) if score >= best: best = score best_model = model return best_model
def resultados_colaborativo(self): """ Método resultados_colaboraivo. Obtiene los resultados del modelo colaborativo. Este método solo se utiliza en la interfaz de texto. """ global train, test, modelo # Se obtienen los resultados precision = precision_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() auc = auc_score(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() recall = recall_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() reciprocal = reciprocal_rank(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() # Se imprimen los resultados imprimir_resultados_clasico(precision, auc, recall, reciprocal)
def recall_at_k_on_ranks(ranks, test_interactions, train_interactions=None, k=10, preserve_rows=False): return recall_at_k( model=ModelMockRanksCacher(ranks.copy()), test_interactions=test_interactions, train_interactions=train_interactions, k=k, preserve_rows=preserve_rows, )
def test_intersections_check(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss="bpr") model.fit_partial(train) # check error is raised when train and test have interactions in common with pytest.raises(ValueError): evaluation.auc_score(model, train, train_interactions=train, check_intersections=True) with pytest.raises(ValueError): evaluation.recall_at_k(model, train, train_interactions=train, check_intersections=True) with pytest.raises(ValueError): evaluation.precision_at_k(model, train, train_interactions=train, check_intersections=True) with pytest.raises(ValueError): evaluation.reciprocal_rank(model, train, train_interactions=train, check_intersections=True) # check no errors raised when train and test have no interactions in common evaluation.auc_score(model, test, train_interactions=train, check_intersections=True) evaluation.recall_at_k(model, test, train_interactions=train, check_intersections=True) evaluation.precision_at_k(model, test, train_interactions=train, check_intersections=True) evaluation.reciprocal_rank(model, test, train_interactions=train, check_intersections=True) # check no error is raised when there are intersections but flag is False evaluation.auc_score(model, train, train_interactions=train, check_intersections=False) evaluation.recall_at_k(model, train, train_interactions=train, check_intersections=False) evaluation.precision_at_k(model, train, train_interactions=train, check_intersections=False) evaluation.reciprocal_rank(model, train, train_interactions=train, check_intersections=False)
def recall_at_k_lightfm(self, model_lightfm, sparse_user_item, k=5): """Recall встроенный в LightFM""" self.recall_res = recall_at_k( model_lightfm, sparse_user_item, user_features=csr_matrix( self.user_feat_lightfm_fixed.values).tocsr(), item_features=csr_matrix( self.item_feat_lightfm_fixed.values).tocsr(), k=k) return self.recall_res
def random_search_recall_at_k(train, test, num_samples=10, num_threads=1,k=5): for hyperparams in itertools.islice(sample_hyperparameters(), num_samples): num_epochs = hyperparams.pop("num_epochs") model = LightFM(**hyperparams) model.fit(train, epochs=num_epochs, num_threads=num_threads) r_at_k_score = recall_at_k(model, test, train_interactions=train, num_threads=num_threads, k=k).mean() hyperparams["num_epochs"] = num_epochs yield (r_at_k_score, hyperparams, model)
def evaluate(self): # Data interactions_train, weights_train, interactions_test, weights_test = \ DataFit.fit_evaluate() new_model = LightFM(loss='warp') new_model.fit(interactions_train, item_features=self.books_features, epochs=100, num_threads=2, sample_weight=weights_train) print('Precision @k(Train): {0}'.format( precision_at_k(new_model, interactions_train).mean())) print('Precision @k(Test): {0}'.format( precision_at_k(new_model, interactions_test).mean())) print('Recall @k(Train): {0}'.format( recall_at_k(new_model, interactions_train).mean())) print('Recall @k(Test): {0}'.format( recall_at_k(new_model, interactions_test).mean())) print('Auc Score(Train): {0}'.format( auc_score(new_model, interactions_train).mean())) print('Auc Score(Test): {0}'.format( auc_score(new_model, interactions_test).mean()))
def test_intersections_check(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss="bpr") model.fit_partial(train) # check error is raised when train and test have interactions in common with pytest.raises(ValueError): evaluation.auc_score( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.recall_at_k( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.precision_at_k( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.reciprocal_rank( model, train, train_interactions=train, check_intersections=True ) # check no errors raised when train and test have no interactions in common evaluation.auc_score( model, test, train_interactions=train, check_intersections=True ) evaluation.recall_at_k( model, test, train_interactions=train, check_intersections=True ) evaluation.precision_at_k( model, test, train_interactions=train, check_intersections=True ) evaluation.reciprocal_rank( model, test, train_interactions=train, check_intersections=True ) # check no error is raised when there are intersections but flag is False evaluation.auc_score( model, train, train_interactions=train, check_intersections=False ) evaluation.recall_at_k( model, train, train_interactions=train, check_intersections=False ) evaluation.precision_at_k( model, train, train_interactions=train, check_intersections=False ) evaluation.reciprocal_rank( model, train, train_interactions=train, check_intersections=False )
def random_search(train, test, item_features=None, user_features=None, num_samples=10, num_threads=16): """ Sample random hyperparameters, fit a LightFM model, and evaluate it on the test set. Parameters ---------- train: np.float32 coo_matrix of shape [n_users, n_items] Training data. test: np.float32 coo_matrix of shape [n_users, n_items] Test data. num_samples: int, optional Number of hyperparameter choices to evaluate. Returns ------- generator of (auc_score, hyperparameter dict, fitted model) """ for hyperparams in itertools.islice(sample_hyperparameters(), num_samples): num_epochs = hyperparams.pop("num_epochs") model = LightFM(**hyperparams) # ,learning_rate=.03 model.fit(train, epochs=num_epochs, num_threads=num_threads, item_features=item_features, user_features=user_features) ### should i pass in train_interactions (when i have repeats) ? # score = auc_score(model, test, train_interactions=train, num_threads=num_threads).mean() # ORIG # score = precision_at_k(model, test, num_threads=num_threads,k=4).mean() score = recall_at_k(model, test, num_threads=num_threads, k=90, item_features=item_features, user_features=user_features).mean() hyperparams["num_epochs"] = num_epochs yield (score, hyperparams, model)
def recall_at_k(model, test_interactions, train_interactions=None, k=10, user_features=None, item_features=None, preserve_rows=False, num_threads=1, check_intersections=True): return evaluation.recall_at_k(model, test_interactions=test_interactions, train_interactions=train_interactions, k=k, user_features=user_features, item_features=item_features, preserve_rows=preserve_rows, num_threads=num_threads, check_intersections=check_intersections)
def precrec(): """Evaluates models on Precision@K/Recall@K and also outputs F1 Score. Measure the precision at k metric for a model: the fraction of known positives in the first k positions of the ranked list of results. A perfect score is 1.0. Measure the recall at k metric for a model: the number of positive items in the first k positions of the ranked list of results divided by the number of positive items in the test period. # A perfect score is 1.0. Compute the F1 score, also known as balanced F-score or F-measure: The F1 score can be interpreted as a weighted average of the precision and recall, where an F1 score reaches its best value at 1 and worst score at 0. The relative contribution of precision and recall to the F1 score are equal. """ # train_precision = precision_at_k(model, # train, # k=k, # user_features=user_features if user_features is not None else None, # item_features=item_features if item_features is not None else None, # num_threads=NUM_THREADS).mean() # logger.info(model_name+' training set Precision@%s: %s' % (k, train_precision)) precision = lightfm_precision_at_k( model=model, train_interactions=train, test_interactions=test, k=k, item_features=item_features ).mean() logger.info(model_name+' Precision@%s: %s' % (k, precision)) recall = recall_at_k( model=model, train_interactions=train, test_interactions=test, k=k, item_features=item_features ).mean() logger.info(model_name+' Recall@%s: %s' % (k, recall)) fmeasure = 2*((precision*recall)/(precision+recall)) logger.info(model_name+' F-Measure: %s' % fmeasure)
def resultados_por_contenido(self): """ Método resultados_por_contenido. Obtiene los resultados del modelo basado en contenido. Este método solo se utiliza en la interfaz de texto. """ global train, test, modelo, item_features, user_features # Se obtienen los resultados precision = precision_at_k(modelo, test, train_interactions=train, k=10, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() auc = auc_score(modelo, test, train_interactions=train, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() recall = recall_at_k(modelo, test, train_interactions=train, k=10, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() reciprocal = reciprocal_rank(modelo, test, train_interactions=train, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() # Se imprimen los resultados imprimir_resultados_clasico(precision, auc, recall, reciprocal)
def train(): start = time.time() from lightfm.evaluation import precision_at_k, recall_at_k, auc_score from utils import clean_data, to_sparse, create_user_dict, create_item_dict, fit_mf_model from utils import items_to_user, items_to_item, create_item_emdedding_distance_matrix, users_to_item print('Modules loaded...') training_metrics={} data = pd.read_csv(training_data) piv, cols, interactions_ = to_sparse(data) interactions_.to_csv(interactions, index=True) user_dict_ = create_user_dict(interactions=interactions_) item_dict_ = create_item_dict(df = data, id_col = 'StockCode', name_col = 'Description') with open(user_dict, 'w') as json_file: json.dump(user_dict_, json_file) with open(item_dict, 'w') as json_file: json.dump(item_dict_, json_file) print('Data preparations ready...') mf_model = fit_mf_model(interactions = interactions_, n_components = 140, loss = 'warp', epoch = 10, n_jobs = 6) print('Model fit...') training_metrics["precision_at_3"] = round(precision_at_k(mf_model, piv, k=3).mean()*100) training_metrics["recall_at_3"] = round(recall_at_k(mf_model, piv, k=3).mean()*100) training_metrics["auc_score"]=round(auc_score(mf_model, piv).mean()*100) pickle.dump(mf_model, open(str(model_directory + "/" +"recomender.pkl"), "wb")) print('Model trained & serialized in %.1f seconds' % (time.time() - start)) return jsonify(training_metrics)
def main(): data = fetch_movielens() # check dataset for key, value in data.items(): print(key, type(value), value.shape) print() # get dataset train = data['train'] test = data['test'] # modeling model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=10) p_test = precision_at_k(model, test, k=10, train_interactions=train) r_test = recall_at_k(model, test, k=10, train_interactions=train) a_test = auc_score(model, test, train_interactions=train) print(p_test) print(r_test) print(a_test) print('DONE')
def run_lightfm(ratings, train, test, k_items, dataset): def create_interaction_matrix(df, user_col, item_col, rating_col, norm=False, threshold=None): ''' Function to create an interaction matrix dataframe from transactional type interactions Required Input - - df = Pandas DataFrame containing user-item interactions - user_col = column name containing user's identifier - item_col = column name containing item's identifier - rating col = column name containing user feedback on interaction with a given item - norm (optional) = True if a normalization of ratings is needed - threshold (required if norm = True) = value above which the rating is favorable Expected output - - Pandas dataframe with user-item interactions ready to be fed in a recommendation algorithm ''' interactions = df.groupby([user_col, item_col])[rating_col] \ .sum().unstack().reset_index(). \ fillna(0).set_index(user_col) if norm: interactions = interactions.applymap(lambda x: 1 if x > threshold else 0) return interactions test_interactions = create_interaction_matrix(df=test, user_col='userId', item_col='movieId', rating_col='rating') budget_l = dataset.budget.unique().tolist() gross_l = dataset.gross.unique().tolist() awards_l = dataset.awards.unique().tolist() nom_l = dataset.nominations.unique().tolist() votes_l = dataset.votes.unique().tolist() item_ids = np.unique(train.movieId.astype(int)) print(f'length dataset: {len(dataset)}') dataset = dataset[dataset.movieId.isin(item_ids)] print(f'length dataset: {len(dataset)}') item_features_list = [f'rating_{f}' for f in range(11)] gen = [ 'Action', 'Adventure', 'Animation', "Children's", 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy', 'Film-Noir', 'Horror', 'IMAX', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western' ] # 'unknown' add unknown for movielens100k item_features_list += gen item_features_list += budget_l item_features_list += gross_l item_features_list += awards_l item_features_list += nom_l item_features_list += votes_l item_features = [] for y, x in dataset.iterrows(): genres = x['genres'] tmp_row = (int(x['movieId']), [ x['rating'], x['budget'], x['gross'], x['awards'], x['nominations'], x['votes'] ]) for g in genres: tmp_row[1].append(g) item_features.append(tmp_row) #item_features = [(int(x['movieId']), [x['rating'], z, x['budget'], x['gross'], x['awards'], x['votes']]) for y, x in dataset.iterrows() for z in x['genres']] #x['nominations'] user_ids = np.unique(train.userId) built_dif = Dataset() built_dif.fit_partial(users=user_ids) built_dif.fit_partial(items=item_ids) built_dif.fit_partial(item_features=item_features_list) dataset_item_features = built_dif.build_item_features(item_features) (interactions, weights) = built_dif.build_interactions( ((int(x['userId']), int(x['movieId'])) for y, x in train.iterrows())) modelx = LightFM(no_components=30, loss='bpr', k=15, random_state=1) modelx.fit(interactions, epochs=30, num_threads=4, item_features=dataset_item_features ) #item_features=dataset_item_features test = sparse.csr_matrix(test_interactions.values) test = test.tocoo() num_users, num_items = built_dif.interactions_shape() print('Num users: {}, num_items {}.'.format(num_users, num_items)) prec_list = dict() rec_list = dict() for num_k in k_items: trainprecision = precision_at_k( modelx, test, k=num_k, item_features=dataset_item_features).mean( ) #item_features=dataset_item_features, print('Hybrid training set precision: %s' % trainprecision) trainrecall = recall_at_k(modelx, test, k=num_k, item_features=dataset_item_features).mean( ) #item_features=dataset_item_features print('Hybrid training set recall: %s' % trainrecall) if num_k in prec_list: prec_list[num_k].append(trainprecision) else: prec_list[num_k] = trainprecision if num_k in rec_list: rec_list[num_k].append(trainrecall) else: rec_list[num_k] = trainrecall return prec_list, rec_list
from lightfm import LightFM model = LightFM(loss='warp', random_state=0) model.fit(train_interactions, user_features=users_features, item_features=items_features, epochs=200, num_threads=1) from lightfm.evaluation import recall_at_k from lightfm.evaluation import precision_at_k print("Train recall@7: %.2f" % recall_at_k(model, train_interactions, k=7, user_features=users_features, item_features=items_features).mean()) print("Test recall@7: %.2f" % recall_at_k(model, test_interactions, train_interactions, k=7, user_features=users_features, item_features=items_features).mean()) print("Train precision@7: %.2f" % precision_at_k(model, train_interactions, k=7, user_features=users_features, item_features=items_features).mean()) print("Test precision@7: %.2f" %
#Below are the results of our random optimiaztion, hardcoded as parameters now. #Best score 0.9843319654464722 at bestparams = {'no_components': 59, 'learning_schedule': 'adagrad', 'loss': 'warp', 'learning_rate': 0.08565020895037347, 'item_alpha': 7.345729662383957e-10, 'user_alpha': 4.776609106732949e-09, 'max_sampled': 14, 'random_state':69} model = LightFM(**bestparams) num_epochs = 8 #uncomment below to optimize the number of epochs we train #num_epochs = optimize_epochs(model,cvinteractions,interactions,item_features,15) model.fit(interactions,item_features=item_features,epochs=num_epochs) testscore = auc_score(model,testinteractions,interactions,item_features=item_features).mean() testprec = precision_at_k(model, testinteractions,interactions,item_features=item_features,k=10).mean() testrecall = recall_at_k(model, testinteractions,interactions,item_features=item_features,k=10).mean() print('Test AUC Score = '+str(testscore)) print('Test Precision = '+str(testprec)) print('Test Recall = '+str(testrecall)) #below we have our model predict for a User between 1 and 20. Simply change #the first argument to get predictions for other users. ranks = model.predict(1, np.arange(num_items)) top_items = np.argsort(-ranks) print("Recommendations for user 1:") for i in range(0,11): print('Recommendation #' + str(i+1) + ' is: ' + winedict[str(top_items[i])])
prev_prec = test_precision epoch = 1 while test_precision >= prev_prec: logging.info('Epoch: %s, Test prec: %.2f' % (epoch, test_precision)) prev_prec = test_precision test_precision = precision_at_k(model, test, k=5).mean() # logging.info('Model fit done') # similarItems = similar_items(item_id=sparseDf.getItemIndexById(4105331), item_features=None, model=model) # for tuple in similarItems: # print('Item id: %s, Accuracy: %s' % (sparseDf.getItemIdFromIndex(tuple[0]), tuple[1])) train_precision = precision_at_k(model, train, k=5).mean() logging.info('Train precision computed') test_precision = precision_at_k(model, test, k=5).mean() logging.info('Test precision computed') logging.info('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) # train_auc = auc_score(model, train).mean() logging.info('Train auc computed') test_auc = auc_score(model, test).mean() logging.info('Test auc computed') logging.info('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) train_recall = recall_at_k(model, train, k=5).mean() logging.info('Train recall computed') test_recall = recall_at_k(model, test, k=5).mean() logging.info('Test recall computed') logging.info('Recall: train %.2f, test %.2f.' % (train_recall, test_recall))
def track_model_metrics(model, train_interactions, test_interactions, k=10, no_epochs=100, no_threads=8, show_plot=True, **kwargs): """Function to record model's performance at each epoch, formats the performance into tidy format, plots the performance and outputs the performance data. Args: model (LightFM instance): fitted LightFM model train_interactions (scipy sparse COO matrix): train interactions set test_interactions (scipy sparse COO matrix): test interaction set k (int): number of recommendations, optional no_epochs (int): Number of epochs to run, optional no_threads (int): Number of parallel threads to use, optional **kwargs: other keyword arguments to be passed down Returns: pandas.DataFrame, LightFM model, matplotlib axes: - Performance traces of the fitted model - Fitted model - Side effect of the method """ # initialising temp data storage model_prec_train = [0] * no_epochs model_prec_test = [0] * no_epochs model_rec_train = [0] * no_epochs model_rec_test = [0] * no_epochs # fit model and store train/test metrics at each epoch for epoch in range(no_epochs): model.fit_partial(interactions=train_interactions, epochs=1, num_threads=no_threads, **kwargs) model_prec_train[epoch] = precision_at_k(model, train_interactions, k=k, **kwargs).mean() model_prec_test[epoch] = precision_at_k(model, test_interactions, k=k, **kwargs).mean() model_rec_train[epoch] = recall_at_k(model, train_interactions, k=k, **kwargs).mean() model_rec_test[epoch] = recall_at_k(model, test_interactions, k=k, **kwargs).mean() # collect the performance metrics into a dataframe fitting_metrics = pd.DataFrame( zip(model_prec_train, model_prec_test, model_rec_train, model_rec_test), columns=[ "model_prec_train", "model_prec_test", "model_rec_train", "model_rec_test", ], ) # convert into tidy format fitting_metrics = fitting_metrics.stack().reset_index() fitting_metrics.columns = ["epoch", "level", "value"] # exact the labels for each observation fitting_metrics["stage"] = fitting_metrics.level.str.split("_").str[-1] fitting_metrics["metric"] = fitting_metrics.level.str.split("_").str[1] fitting_metrics.drop(["level"], axis=1, inplace=True) # replace the metric keys to improve visualisation metric_keys = {"prec": "Precision", "rec": "Recall"} fitting_metrics.metric.replace(metric_keys, inplace=True) # plots the performance data if show_plot: model_perf_plots(fitting_metrics) return fitting_metrics, model
def run(self, epochs: int = 1, no_components: int = 50, learning_rate: float = 0.05) -> Dict[str, float]: """ build interaction matrix -> build movie features -> build model Example (5000 samples, 50 components, 5 epochs, learning_rate=0.05) ================================= {'auc_train': 0.66268414, 'auc_test': 0.67257625, 'precision_train@10': 0.035984848, 'precision_test@10': 0.014193548, 'recall_train@10': 0.06827082513973247, 'recall_test@10': 0.0646373101211811} ########################### #### Random Stratified #### ########################### Example (2 million samples, 50 components, 1 epochs, learning_rate=0.05) ================================= {'auc_train': 0.5171841, 'auc_test': 0.51610065, 'precision_train@10': 0.018248174, 'precision_test@10': 0.0040145987, 'recall_train@10': 0.0008001067196610589, 'recall_t0.018248174est@10': 0.0007001527280332769} ######################## #### Popular Active #### ######################## Example (333000 samples, 150 components, 1 epochs, learning_rate=0.05) 20% test data ================================= {'auc_train': 0.63388383, 'auc_test': 0.5569484, 'precision_train@10': 0.7255412, 'precision_test@10': 0.17099567, 'recall_train@10': 0.006322884137545113, 'recall_test@10': 0.006053869700910709} Example (333000 samples, 50 components, 1 epochs, learning_rate=0.05) 40% test data ================================= {'auc_train': 0.6001097, 'auc_test': 0.56429684, 'precision_train@10': 0.56060606, 'precision_test@10': 0.33030304, 'recall_train@10': 0.006517918240037026, 'recall_test@10': 0.005792534657980192} Example (333000 samples, 50 components, 20 epochs, learning_rate=0.05) 40% test data ================================= {'auc_train': 0.6077434, 'auc_test': 0.5688331, 'precision_train@10': 0.5874459, 'precision_test@10': 0.32424247, 'recall_train@10': 0.0068082500065638684, 'recall_test@10': 0.005756504594433489} Example (333000 samples, 50 components, 1 epochs, learning_rate=0.05) 40% test data with normalization ================================= {'auc_train': 0.60080063, 'auc_test': 0.56425303, 'precision_train@10': 0.56926405, 'precision_test@10': 0.33679655, 'recall_train@10': 0.006628036812872702, 'recall_test@10': 0.005913302996971047} """ ## Build Matrix Factorization between Customer and Movie data = self._filter_data dataset = Dataset() dataset.fit(data['Cust_Id'].unique(), data['Movie_Id'].unique(), item_features=self.get_combination) (interactions, weights) = dataset.build_interactions([ (x['Cust_Id'], x['Movie_Id'], x['Rating']) for index, x in data.iterrows() ]) train, test = random_train_test_split( interactions, test_percentage=0.4, random_state=np.random.RandomState(7)) print("Finished creating interactions matrix!") ## Build movie features movies_id, tfidf_data = self.get_tfidf_matrix features_lists = [list(x) for x in tfidf_data.values] movies_features = dataset.build_item_features( data=self.get_movies_tuple(features_lists, movies_id, tfidf_data), normalize=True) print("Finished building movie features!") ## Build model model = LightFM(no_components=no_components, learning_rate=learning_rate, loss='warp', k=15) model.fit(train, epochs=epochs, item_features=movies_features, num_threads=4) print("Finished building LightFM model!") with open('hybrid_model_popular_active.pickle', 'wb') as fle: pickle.dump(model, fle, protocol=pickle.HIGHEST_PROTOCOL) print("Finished saving LightFM model!") return { "auc_train": auc_score(model, train, item_features=movies_features).mean(), "auc_test": auc_score(model, test, item_features=movies_features).mean(), "precision_train@10": precision_at_k(model, train, item_features=movies_features, k=10).mean(), "precision_test@10": precision_at_k(model, test, item_features=movies_features, k=10).mean(), "recall_train@10": recall_at_k(model, train, item_features=movies_features, k=10).mean(), "recall_test@10": recall_at_k(model, test, item_features=movies_features, k=10).mean() }
#with open(MODEL_CHECKPOINT_PATH, 'wb') as fle: # pickle.dump(model, fle, protocol=pickle.HIGHEST_PROTOCOL) train_precision = precision_at_k(model, data['train'], k=10, item_features=item_features).mean() test_precision = precision_at_k(model, data['test'], k=10, train_interactions=data['train'], item_features=item_features).mean() train_recall = recall_at_k(model, data['train'], k=10, item_features=item_features).mean() test_recall = recall_at_k(model, data['test'], k=10, train_interactions=data['train'], item_features=item_features).mean() train_auc = auc_score(model, data['train'], item_features=item_features).mean() test_auc = auc_score(model, data['test'], train_interactions=data['train'], item_features=item_features).mean()
def obtener_metricas_gui(self): """ Método obtener_metricas_gui. Obtiene las métricas del modelo escogido. Este método solo se utiliza en la interfaz web. Returns ------- metricas_devueltas: dict diccionario con las métricas del modelo """ global train, test, modelo, item_features, user_features # Se guardan las métricas en un diccionario para su futura muestra en la interfaz web metricas = dict() # Se calculan las métricas if self.opcion_modelo == 1: precision = precision_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() auc = auc_score(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() recall = recall_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() reciprocal = reciprocal_rank(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() else: precision = precision_at_k(modelo, test, train_interactions=train, k=10, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() auc = auc_score(modelo, test, train_interactions=train, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() recall = recall_at_k(modelo, test, train_interactions=train, k=10, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() reciprocal = reciprocal_rank(modelo, test, train_interactions=train, user_features=user_features, item_features=item_features, num_threads=self.CPU_THREADS).mean() # Se guardan las métricas en el diccionario y se formatea su salida metricas_devueltas = { "Precisión k": format(precision, '.4f'), "AUC Score": format(auc, '.4f'), "Recall k": format(recall, '.4f'), "Ranking recíproco": format(reciprocal, '.4f') } metricas_a_guardar = { "Precisión k": [format(precision, '.4f')], "AUC Score": [format(auc, '.4f')], "Recall k": [format(recall, '.4f')], "Ranking recíproco": [format(reciprocal, '.4f')] } # Se guardan las métricas en un archivo .csv guardar_resultados(metricas_a_guardar) return metricas_devueltas