def evaluate_fm(model_, te_, tr_, items_features=None, users_features=None): if not tr_.multiply(te_).nnz == 0: print('train test interaction are not fully disjoin') # Compute and print the AUC score train_auc = auc_score(model_, tr_, item_features=items_features, user_features=users_features, num_threads=NUM_THREADS).mean() print('Collaborative filtering train AUC: %s' % train_auc) test_auc = auc_score(model_, te_, train_interactions=tr_, item_features=items_features, user_features=users_features, num_threads=NUM_THREADS).mean() print('Collaborative filtering test AUC: %s' % test_auc) p_at_k_train = precision_at_k(model_, tr_, item_features=items_features, user_features=users_features, k=5, num_threads=NUM_THREADS).mean() p_at_k_test = precision_at_k(model_, te_, train_interactions=tr_, item_features=items_features, user_features=users_features, k=5, num_threads=NUM_THREADS).mean() print("Train precision: %.2f" % p_at_k_train) print("Test precision: %.2f" % p_at_k_test)
def auc(): """Evaluates models on the ROC AUC metric. Measure the ROC AUC metric for a model: the probability that a randomly chosen positive example has a higher score than a randomly chosen negative example. A perfect score is 1.0. """ auc = auc_score(model=model, train_interactions=train, test_interactions=test, item_features=item_features).mean() logger.info(model_name + ' AUC: %s' % auc) train_auc = auc_score(model, train, item_features=item_features if item_features is not None else None, num_threads=NUM_THREADS).mean() logger.info(model_name + ' training set AUC: %s' % train_auc) test_auc = auc_score(model, test, train_interactions=train, item_features=item_features if item_features is not None else None, num_threads=NUM_THREADS).mean() logger.info(model_name + ' test set AUC: %s' % auc)
def train_model(train, test, user_features, item_features): log.info("Initializing model") model = LightFM(loss="warp", item_alpha=ITEM_ALPHA, no_components=NUM_COMPONENTS) log.info("Training model") model = model.fit(train, user_features=user_features, item_features=item_features, epochs=NUM_EPOCHS, num_threads=NUM_THREADS) log.info("Scoring") train_auc = auc_score(model, train, user_features=user_features, item_features=item_features, num_threads=NUM_THREADS).mean() log.info(f"Training set AUC: {train_auc}") test_auc = auc_score(model, test, train_interactions=train, user_features=user_features, item_features=item_features, num_threads=NUM_THREADS).mean() log.info(f"Test set AUC: {test_auc}") return model
def main(): movielens = fetch_movielens() train = movielens['train'] test = movielens['test'] print(train.shape) print(test.shape) model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=5) k = 10 train_recall = recall_at_k(model, train, k=k).mean() test_recall = recall_at_k(model, test, k=k).mean() print(f'recall_at_{k}(train): {train_recall}') print(f'recall_at_{k}(test) : {test_recall}') train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print(f'auc_score(train): {train_auc}') print(f'auc_score(test) : {test_auc}') y_train_preds = model.predict_rank(train) y_test_preds = model.predict_rank(test) train_dcg = dcg_score(train.toarray(), y_train_preds.toarray()) test_dcg = dcg_score(test.toarray(), y_test_preds.toarray()) print(f'dcg_score(train): {train_dcg}') print(f'dcg_score(test) : {test_dcg}') print('DONE') return 0
def train(data, user_features=None, item_features=None, use_features=False): loss_type = "warp" # "bpr" model = LightFM(learning_rate=0.05, loss=loss_type, max_sampled=100) if use_features: model.fit_partial(data, epochs=20, user_features=friends_features, item_features=item_features) train_precision = precision_at_k(model, data, k=10, user_features=friends_features, item_features=item_features).mean() train_auc = auc_score(model, data, user_features=friends_features, item_features=item_features).mean() print(f'Precision: train {train_precision:.2f}') print(f'AUC: train {train_auc:.2f}') else: model.fit_partial(data, epochs=20) train_precision = precision_at_k(model, data, k=10).mean() train_auc = auc_score(model, data).mean() print(f'Precision: train {train_precision:.2f}') print(f'AUC: train {train_auc:.2f}') return model
def evaluate(model, train, test, hybrid=False, features=None): if hybrid: auc_train = np.mean(auc_score(model, train, item_features=features)) pre_train = np.mean(precision_at_k(model, train, item_features=features)) mrr_train = np.mean(reciprocal_rank(model, train, item_features=features)) auc_test = np.mean(auc_score(model, test, item_features=features)) pre_test = np.mean(precision_at_k(model, test, item_features=features)) mrr_test = np.mean(reciprocal_rank(model, test, item_features=features)) else: auc_train = np.mean(auc_score(model, train)) pre_train = np.mean(precision_at_k(model, train)) mrr_train = np.mean(reciprocal_rank(model, train)) auc_test = np.mean(auc_score(model, test)) pre_test = np.mean(precision_at_k(model, test)) mrr_test = np.mean(reciprocal_rank(model, test)) res_dict = {'auc_train': auc_train, 'pre_train': pre_train, 'mrr_train': mrr_train, 'auc_test': auc_test, 'pre_test': pre_test, 'mrr_test': mrr_test} print('The AUC Score is in training/validation: ', auc_train,' / ', auc_test) print('The mean precision at k Score in training/validation is: ', pre_train, ' / ', pre_test) print('The mean reciprocal rank in training/validation is: ', mrr_train, ' / ', mrr_test) print('_________________________________________________________') return res_dict
def test_auc_score(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss='bpr') model.fit_partial(train) auc = evaluation.auc_score(model, test, num_threads=2) expected_auc = np.array(_auc(model, test)) assert auc.shape == expected_auc.shape assert np.abs(auc.mean() - expected_auc.mean()) < 0.01 assert len(auc) == (test.getnnz(axis=1) > 0).sum() assert len(evaluation.auc_score(model, train, preserve_rows=True)) == test.shape[0] # With omitting train interactions auc = evaluation.auc_score(model, test, train_interactions=train, num_threads=2) expected_auc = np.array(_auc(model, test, train)) assert np.abs(auc.mean() - expected_auc.mean()) < 0.01
def lightfm_model(data, prec_at_k=100, train_split=0.8, epochs=10): """ Code to evaluate LightFm model Data is a scipy sparse matrix https://arxiv.org/abs/1507.08439 """ model = LightFM(learning_rate=0.05, loss='logistic') train, test = random_train_test_split(data, test_percentage=1 - train_split) model.fit(train, epochs=epochs) #, num_threads=1) train_precision = precision_at_k(model, train, k=prec_at_k) test_precision = precision_at_k(model, test, k=prec_at_k, train_interactions=train) train_auc = auc_score(model, train) test_auc = auc_score(model, test, train_interactions=train) print('Performance of LightFm Model \n') print( f'Precision \t Train: {train_precision.mean():.2f} \t Test: {test_precision.mean():.2f}' ) print( f'AUC \t\t Train: {train_auc.mean():.2f} \t Test: {test_auc.mean():.2f}' ) return (train_auc, test_auc, train_precision, test_precision, prec_at_k)
def test_auc_score(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss='bpr') model.fit_partial(train) auc = evaluation.auc_score(model, test, num_threads=2) expected_auc = np.array(_auc(model, test)) assert auc.shape == expected_auc.shape assert np.abs(auc.mean() - expected_auc.mean()) < 0.01 assert len(auc) == (test.getnnz(axis=1) > 0).sum() assert len(evaluation.auc_score(model, train, preserve_rows=True)) == test.shape[0] # With omitting train interactions auc = evaluation.auc_score(model, test, train_interactions=train, num_threads=2) expected_auc = np.array(_auc(model, test, train)) assert np.abs(auc.mean() - expected_auc.mean()) < 0.01
def main(): movielens = fetch_movielens() train = movielens['train'] print(type(train)) print(train.toarray()[:5, :]) test = movielens['test'] print(type(test)) print(test.toarray()[:5, :]) model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=10) train_precision = precision_at_k(model, train, k=10).mean() test_precision = precision_at_k(model, test, k=10, train_interactions=train).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test, train_interactions=train).mean() print(f'train precision: {train_precision}') print(f'test precision: {test_precision}') print(f'train auc: {train_auc}') print(f'test auc: {test_auc}') print('DONE')
def evaluate(self, model, train, test, k=10): train_precision = precision_at_k(model, train, k=k).mean() test_precision = precision_at_k(model, test, k=k).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() return train_precision, test_precision, train_auc, test_auc
def validate_item_features(ctx, data_home): data = fetch_stackexchange('crossvalidated', test_set_fraction=0.1, indicator_features=False, tag_features=True, data_home=data_home) train = data['train'] test = data['test'] # Set the number of threads; you can increase this # ify you have more physical cores available. NUM_COMPONENTS = 30 NUM_EPOCHS = 3 ITEM_ALPHA = 1e-6 # Let's fit a WARP model: these generally have the best performance. model = LightFM(loss='warp', item_alpha=ITEM_ALPHA, no_components=NUM_COMPONENTS) # Run 3 epochs and time it. model = model.fit(train, epochs=NUM_EPOCHS) train_auc = auc_score(model, train).mean() print('Collaborative filtering train AUC: %s' % train_auc) test_auc = auc_score(model, test, train_interactions=train).mean() print('Collaborative filtering test AUC: %s' % test_auc) # Set biases to zero model.item_biases *= 0.0 test_auc = auc_score(model, test, train_interactions=train).mean() print('Collaborative filtering test AUC: %s' % test_auc) item_features = data['item_features'] tag_labels = data['item_feature_labels'] print('There are %s distinct tags, with values like %s.' % (item_features.shape[1], tag_labels[:3].tolist())) # Define a new model instance model = LightFM(loss='warp', item_alpha=ITEM_ALPHA, no_components=NUM_COMPONENTS) # Fit the hybrid model. Note that this time, we pass # in the item features matrix. model = model.fit(train, item_features=item_features, epochs=NUM_EPOCHS) # Don't forget the pass in the item features again! train_auc = auc_score(model, train, item_features=item_features).mean() print('Hybrid training set AUC: %s' % train_auc) test_auc = auc_score(model, test, train_interactions=train, item_features=item_features).mean() print('Hybrid test set AUC: %s' % test_auc)
def lightfm_model(data, prec_at_k=10, train_split=0.8): """ Code to evaluate LightFm model Data is a scipy sparse matrix https://arxiv.org/abs/1507.08439 """ model = LightFM(learning_rate=0.05, loss='bpr') train, test = random_train_test_split(data, test_percentage=1 - train_split) model.fit(train, epochs=10) train_precision = precision_at_k(model, train, k=10) test_precision = precision_at_k(model, test, k=10, train_interactions=train) train_auc = auc_score(model, train) test_auc = auc_score(model, test, train_interactions=train) print('Performance of LightFm Model \n') print( f'Precision \t Train: {train_precision.mean():.2f} \t Test: {test_precision.mean():.2f}' ) print( f'AUC \t\t Train: {train_auc.mean():.2f} \t Test: {test_auc.mean():.2f}' ) fig, ax = plt.subplots(2, 2, figsize=(15, 10)) ax[0, 0].hist(train_auc, bins='auto') ax[0, 0].title.set_text('Distribution of Train AUC score over users') ax[0, 0].set_ylabel('Count') ax[0, 0].set_xlabel('AUC Score') ax[0, 1].hist(test_auc, bins='auto') ax[0, 1].title.set_text('Distribution of Test AUC score over users') ax[0, 1].set_ylabel('Count') ax[0, 1].set_xlabel('AUC Score') ax[1, 0].hist(train_precision, bins='auto') ax[1, 0].title.set_text( f'Distribution of Train Precision @ {prec_at_k} for all users') ax[1, 0].set_ylabel('Count') ax[1, 0].set_xlabel(f'Precision @ {prec_at_k}') ax[1, 1].hist(test_precision, bins='auto') ax[1, 1].title.set_text( f'Distribution of Test Precision @ {prec_at_k} for all users') ax[1, 1].set_ylabel('Count') ax[1, 1].set_xlabel(f'Precision @ {prec_at_k}') plt.show() print('\n')
def collab_filtering(): """ implements collaborative filtering version by using only the rating data from movielens dataset :return: """ data = fetch_movielens() for key, value in data.items(): print(key, type(value), value.shape) train = data['train'] test = data['test'] print( 'The dataset has %s users and %s items, ' 'with %s interactions in the test and %s interactions in the training set.' % (train.shape[0], train.shape[1], test.getnnz(), train.getnnz())) model = LightFM(learning_rate=0.05, loss='bpr') model.fit(train, epochs=50, num_threads=5) train_precision = precision_at_k(model, train, k=10).mean() test_precision = precision_at_k(model, test, k=10).mean() train_recall = recall_at_k(model, test, k=10).mean() test_recall = recall_at_k(model, test, k=10).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) print('Recall: train %.2f, test %.2f.' % (train_recall, test_recall)) print('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) model = LightFM(learning_rate=0.05, loss='warp') #resume training from the model's previous state model.fit_partial(train, epochs=50, num_threads=5) train_precision = precision_at_k(model, train, k=10).mean() test_precision = precision_at_k(model, test, k=10).mean() train_recall = recall_at_k(model, test, k=10).mean() test_recall = recall_at_k(model, test, k=10).mean() train_auc = auc_score(model, train).mean() test_auc = auc_score(model, test).mean() print("*****************") print("After re-training") print('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) print('Recall: train %.2f, test %.2f.' % (train_recall, test_recall)) print('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) #check sample recommendation sample_recommendation(model, data, [3, 25, 450])
def eval(model, train, val): # auc print("Train auc: %.2f" % auc_score(model, train).mean()) print("Val auc: %.2f" % auc_score(model, val).mean()) # precision_at_k print("Train precision: %.2f" % precision_at_k(model, train, k=5).mean()) print("Val precision: %.2f" % precision_at_k(model, val, k=5).mean()) # recall_at_k print("Train recall: %.2f" % precision_at_k(model, train, k=5).mean()) print("Val recall: %.2f" % precision_at_k(model, val, k=5).mean())
def evaluate_model(model, train, test, item_fetures=None, user_features=None, num_threads=1): train_precision = precision_at_k(model, train, k=10, user_features=user_features, item_features=item_fetures, num_threads=num_threads).mean() test_precision = precision_at_k(model, test, train_interactions=train, k=10, user_features=user_features, item_features=item_fetures, num_threads=num_threads).mean() train_auc = auc_score(model, train, user_features=user_features, item_features=item_fetures, num_threads=num_threads).mean() test_auc = auc_score(model, test, train_interactions=train, user_features=user_features, item_features=item_fetures, num_threads=num_threads).mean() print('Precision: train %.2f, test %.2f.' % (train_precision, test_precision)) print('AUC: train %.2f, test %.2f.' % (train_auc, test_auc)) return train_precision, test_precision, train_auc, test_auc
def patk_learning_curve(model, train, test, iterarray, user_features=None, item_features=None, k=5, **fit_params): old_epoch = 0 train_patk = [] test_patk = [] warp_duration = [] # bpr_duration = [] train_warp_auc = [] test_warp_auc = [] # bpr_auc = [] headers = ['Epoch', 'train p@5', 'train_auc', 'test p@5', 'test_auc'] print_log(headers, header=True) for epoch in iterarray: more = epoch - old_epoch start = time.time() model.fit_partial(train, user_features=user_features, epochs=more, item_features=item_features, **fit_params) warp_duration.append(time.time() - start) train_warp_auc.append( auc_score(model, train, item_features=item_features).mean()) test_warp_auc.append( auc_score(model, test, item_features=item_features, train_interactions=train).mean()) this_test = precision_at_k(model, test, train_interactions=train, item_features=item_features, k=k) this_train = precision_at_k(model, train, train_interactions=None, item_features=item_features, k=k) train_patk.append(np.mean(this_train)) test_patk.append(np.mean(this_test)) row = [ epoch, train_patk[-1], train_warp_auc[-1], test_patk[-1], test_warp_auc[-1] ] print_log(row) return model, train_patk, test_patk, warp_duration, train_warp_auc, test_warp_auc
def model_train_test(spark, train, val, test, no_components=20, learning_rate=0.01, epochs=15, k=10): ''' Function to train Lightfm collaborative filtering recommender system and test on validation data and testing data Parameters ---------- spark : spark session object train: Type-Sparse Matrix: Processed training data val: Type-Sparse Matrix: Processed validation data test: Type-Sparse Matrix: Processed testing data No_components: type-Int: latent factors learning_rate: type-int: learning rate epochs: type-int: Iterations k: type-int: Top-k predictions for every user Return ---------- None ''' start_time = time.time() #Create Lightfm object model = LightFM(no_components=no_components, learning_rate=learning_rate, loss='warp') #Fit model model.fit(train, epochs=epochs, num_threads=1) #Record time (Fit time) fit_time = time.time() #Calculate AUC value auc_val = auc_score(model, val).mean() score_calc_time = time.time() auc_test = auc_score(model, test).mean() auc_train = auc_score(model, train).mean() #Calculate Precision_at_k P_at_K = precision_at_k(model, test, k=k) precision_value = np.mean(P_at_K) print("For no_components = {}, learning_rate = {} ".format( no_components, learning_rate)) print("Train AUC Score: {}".format(auc_train)) print("Val AUC Score: {}".format(auc_val)) print("Test AUC Score: {}".format(auc_test)) print("Precision at k={} Score: {}".format(k, precision_value)) print("--- Fit time: {} mins ---".format(fit_time - start_time)) print("--- Score time: {} mins ---".format(score_calc_time - fit_time))
def evaluate_model(df, user_id_col='user_id', item_id_col='business_id', stratify=None): """ Model evaluation. Args: df: the input dataframe. user_id_col: user id column. item_id_col: item id column. stratify: if use stratification. Returns: train_auc: training set auc score. test_auc: testing set auc score. """ # model evaluation # create test and train datasets print('model evaluation') train, test = train_test_split(df, test_size=0.2, stratify=stratify) ds = Dataset() # we call fit to supply userid, item id and user/item features ds.fit( df[user_id_col].unique(), # all the users df[item_id_col].unique(), # all the items ) # plugging in the interactions (train_interactions, train_weights) = ds.build_interactions([ (x[0], x[1], x[2]) for x in train.values ]) (test_interactions, _) = ds.build_interactions([(x[0], x[1], x[2]) for x in test.values]) # model model = LightFM(no_components=100, learning_rate=0.05, loss='warp', max_sampled=50) model.fit(train_interactions, sample_weight=train_weights, epochs=10, num_threads=10) # auc-roc train_auc = auc_score(model, train_interactions, num_threads=20).mean() print('Training set AUC: %s' % train_auc) test_auc = auc_score(model, test_interactions, num_threads=20).mean() print('Testing set AUC: %s' % test_auc)
def measure_accuracies(model, data): print("\nMeasuring accuracies of the model...") # evaluate the precision@k metric training_precision = precision_at_k(model, data["train"], k=PRECISION_K).mean() test_precision = precision_at_k(model, data["test"], k=PRECISION_K).mean() # evaluate the AUROC metric training_auc = auc_score(model, data["train"]).mean() test_auc = auc_score(model, data["test"]).mean() # print them out print("Precision@k: training %.2f, test %.2f" % (training_precision, test_precision)) print("AUC: training %.2f, test %.2f" % (training_auc, test_auc))
def _get_metrics(model, train_set, test_set): train_set = train_set.tocsr() test_set = test_set.tocsr() train_set.data[train_set.data < 0] = 0.0 test_set.data[test_set.data < 0] = 0.0 train_set.eliminate_zeros() test_set.eliminate_zeros() return (precision_at_k(model, train_set).mean(), precision_at_k(model, test_set).mean(), auc_score(model, train_set).mean(), auc_score(model, test_set).mean())
def evaluate_model(model, metric, test, train): """ Evaluate trained model on the test set, using one of the three available accuracy metrics AUC: the probability that a randomly chosen positive example has a higher score than a randomly chosen negative example. Precision: the fraction of known positives in the first k positions of the ranked list of results. Recall: the number of positive items in the first k positions of the ranked list of results divided by the number of positive items in the test period. :param model:(LightFM, required) - model to be evaluated :param metric:(string, required) - accuracy metric to be used, one of ['auc', 'precision', 'recall'] :param test:(COO matrix, required) - known positives used to test the model :param train:(COO matrix, required) - training set; these interactions will be omitted from the score calculations to avoid re-recommending known positives. :return: test_score (float) - score computed on the test set """ try: # make sure the metric is correct assert metric in ['auc', 'precision', 'recall'] if metric == 'auc': test_score = auc_score(model, test, train).mean() elif metric == 'precision': test_score = precision_at_k(model, test, train, k=5).mean() else: test_score = recall_at_k(model, test, train, k=5).mean() return test_score except AssertionError: print('The metric provided is not correct or available!')
def Model_fit_part1(train, test): # initialising model with warp loss function model_without_features = LightFM(loss="warp") start = time.time() #=================== model_without_features.fit(train, user_features=None, item_features=None, sample_weight=None, epochs=1, num_threads=4, verbose=False) #=================== end = time.time() print("time taken = {0:.{1}f} seconds".format(end - start, 2)) print("checking accuracy and results with test data") # auc metric score (ranging from 0 to 1) start = time.time() #=================== auc_without_features = auc_score(model=model_without_features, test_interactions=test, num_threads=4, check_intersections=False) print(auc_without_features) #=================== end = time.time() print("accurcay model time taken = {0:.{1}f} seconds".format( end - start, 2))
def Model_fit_part2(train, prod_features, test): # initialising model with warp loss function from lightfm import LightFM from lightfm.evaluation import auc_score model_with_features = LightFM(loss="warp") # fitting the model with hybrid collaborative filtering + content based (product + features) start = time.time() #=================== model_with_features.fit(train, user_features=None, item_features=prod_features, sample_weight=None, epochs=1, num_threads=4, verbose=False) auc_with_features = auc_score(model=model_with_features, test_interactions=test, train_interactions=train, item_features=prod_features, num_threads=4, check_intersections=False) #=================== end = time.time() print("time taken = {0:.{1}f} seconds".format(end - start, 2)) print("average AUC without adding item-feature interaction = {0:.{1}f}". format(auc_with_features.mean(), 2))
def testAUC(self): self.train_auc = auc_score(self.model, self.item_user, item_features=self.item, user_features=self.user, num_threads=self.num_threads).mean() print('Hybrid testing set AUC: %s' % self.train_auc)
def test_LightFM_model(model, test_interactions, train_interactions, user_features, movie_features, k=5): test_precision = precision_at_k(model, test_interactions, train_interactions, k=k, user_features=user_features, item_features=movie_features, num_threads=2).mean() test_recall = recall_at_k(model, test_interactions, train_interactions, k=k, user_features=user_features, item_features=movie_features, num_threads=2).mean() test_auc = auc_score(model, test_interactions, train_interactions, user_features=user_features, item_features=movie_features, num_threads=2).mean() print('Model') print('Precision at k=', str(k), ': ', round(test_precision, 3), sep='') print('Recall at k=', str(k) + ': ', round(test_recall, 3), sep='') print('AUC: ', round(test_auc, 3), sep='') return ({ 'precision': round(test_precision, 3), 'recall': round(test_recall, 3), 'auc': round(test_auc, 3) })
def runMF(product_type, num_samples=20, num_threads=2): """ 헤비유저를 위한 화장품 추천 모델을 만들기 위한 함수입니다. LightFM 모델을 사용합니다. sample_hyperparameters 함수를 이용하여 하이퍼 파라미터를 구해서 모델 파라미터에 적용시켜 모델을 만들어 줍니다. ------- interactions:유저와 화장품 정보로 만든 희소 행렬 num_samples: 하이퍼 파라미터를 구할때 만들어지는 랜덤한 숫자들을 지정한 개수만큼만 나오게 합니다 """ interactions=pickle.load(open("./pickle_data/"+product_type+"/interactions.p", "rb")) user_features=pickle.load(open("./pickle_data/user_features.p", "rb")) x = sparse.csr_matrix(interactions.values) train, test = random_train_test_split(x, test_percentage=0.2, random_state=RandomState(523)) #itertools 패키지의 islice기능을 사용하여 위에서 설정한 파라미터값을 설정한 num_samples값만큼 반복적으로 나오게하여 적용합니다. for hyperparams in itertools.islice(sample_hyperparameters(), num_samples): num_epochs = hyperparams.pop("num_epochs") model = LightFM(**hyperparams) model.fit(train, user_features, epochs=num_epochs, num_threads=num_threads, verbose=True) auc = auc_score(model, test, train_interactions = train, num_threads=num_threads,user_features=user_features).mean() hyperparams["num_epochs"] = num_epochs yield (auc, model)
def run(self): self.random = RandomState(self.random_seed) orders_path = self.requires()['orders'].output().path _, features, interactions = self._generate_matrices(orders_path) train_features, val_features, train_interactiosn, val_interactions = \ train_test_split(features, interactions, test_size=0.1, random_state=self.random) model = LightFM(loss='logistic', no_components=self.no_components, random_state=self.random) wait = 0 best_val_auc = None for epoch in range(1, self.epochs + 1): model.fit_partial(train_interactiosn, user_features=train_features, epochs=self.epochs, num_threads=self.num_threads) auc_scores = auc_score(model, val_interactions, user_features=val_features, num_threads=self.num_threads) current_val_auc = np.nan_to_num(auc_scores).mean() if best_val_auc is None or current_val_auc > best_val_auc: joblib.dump(model, self.output().path) best_val_auc = current_val_auc wait = 0 else: wait += 1 if wait == self.patience: break print('Epoch {}/{} - AUC: {:.6g}'.format(epoch, self.epochs, best_val_auc))
def resultados_colaborativo(self): """ Método resultados_colaboraivo. Obtiene los resultados del modelo colaborativo. Este método solo se utiliza en la interfaz de texto. """ global train, test, modelo # Se obtienen los resultados precision = precision_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() auc = auc_score(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() recall = recall_at_k(modelo, test, train_interactions=train, k=10, num_threads=self.CPU_THREADS).mean() reciprocal = reciprocal_rank(modelo, test, train_interactions=train, num_threads=self.CPU_THREADS).mean() # Se imprimen los resultados imprimir_resultados_clasico(precision, auc, recall, reciprocal)
def best_reccomendation(): #define variables best = 0.0 best_model = '' for model in models: score = 0.0 pak_score = evaluation.precision_at_k(model, data2['test']) score += np.mean(pak_score) rak_score = evaluation.recall_at_k(model, data2['test']) score += np.mean(rak_score) auc_score = evaluation.auc_score(model, data2['test']) score += np.mean(auc_score) rr_score = evaluation.reciprocal_rank(model, data2['test']) score += np.mean(rr_score) print(score) if score >= best: best = score best_model = model return best_model
def plot_roc(data): alpha = 1e-05 epochs = 50 num_components = 32 warp_model = LightFM(no_components=num_components, loss='warp', learning_schedule='adagrad', max_sampled=3, user_alpha=alpha, item_alpha=alpha) bpr_model = LightFM(no_components=num_components, loss='bpr', learning_schedule='adagrad', user_alpha=alpha, item_alpha=alpha) logistic_model=LightFM(no_components=num_components, loss='logistic', learning_schedule='adagrad', user_alpha=alpha, item_alpha=alpha) warp_auc = [] bpr_auc = [] logistic_auc = [] for epoch in range(epochs): warp_model.fit_partial(data['matrix'], epochs=5) warp_auc.append(auc_score(warp_model, data['matrix']).mean()) for epoch in range(epochs): bpr_model.fit_partial(data['matrix'], epochs=5) bpr_auc.append(auc_score(bpr_model, data['matrix']).mean()) for epoch in range(epochs): logistic_model.fit_partial(data['matrix'], epochs=5) logistic_auc.append(auc_score(bpr_model, data['matrix']).mean()) x = np.arange(epochs) plt.plot(x, np.array(warp_auc)) plt.plot(x, np.array(bpr_auc)) plt.plot(x, np.array(logistic_auc)) plt.legend(['WARP AUC', 'BPR AUC', 'LOGISTIC AUC'], loc='upper right') return plt.show(block=False)
def _get_metrics(model, train_set, test_set): train_set = train_set.tocsr() test_set = test_set.tocsr() train_set.data[train_set.data < 0] = 0.0 test_set.data[test_set.data < 0] = 0.0 train_set.eliminate_zeros() test_set.eliminate_zeros() train_users = train_set.getnnz(axis=1) > 0 test_users = test_set.getnnz(axis=1) > 0 return (precision_at_k(model, train_set)[train_users].mean(), precision_at_k(model, test_set)[test_users].mean(), auc_score(model, train_set)[train_users].mean(), auc_score(model, test_set)[test_users].mean())
def test_intersections_check(): no_users, no_items = (10, 100) train, test = _generate_data(no_users, no_items) model = LightFM(loss="bpr") model.fit_partial(train) # check error is raised when train and test have interactions in common with pytest.raises(ValueError): evaluation.auc_score( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.recall_at_k( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.precision_at_k( model, train, train_interactions=train, check_intersections=True ) with pytest.raises(ValueError): evaluation.reciprocal_rank( model, train, train_interactions=train, check_intersections=True ) # check no errors raised when train and test have no interactions in common evaluation.auc_score( model, test, train_interactions=train, check_intersections=True ) evaluation.recall_at_k( model, test, train_interactions=train, check_intersections=True ) evaluation.precision_at_k( model, test, train_interactions=train, check_intersections=True ) evaluation.reciprocal_rank( model, test, train_interactions=train, check_intersections=True ) # check no error is raised when there are intersections but flag is False evaluation.auc_score( model, train, train_interactions=train, check_intersections=False ) evaluation.recall_at_k( model, train, train_interactions=train, check_intersections=False ) evaluation.precision_at_k( model, train, train_interactions=train, check_intersections=False ) evaluation.reciprocal_rank( model, train, train_interactions=train, check_intersections=False )
def test_auc_score(): no_users, no_items = (10, 100) train = sp.rand(no_users, no_items, format='coo') train.data = np.ones_like(train.data) model = LightFM(loss='bpr') model.fit_partial(train) auc = evaluation.auc_score(model, train, num_threads=2)[train.getnnz(axis=1) > 0] expected_auc = np.array(_auc(model, train)) assert auc.shape == expected_auc.shape assert np.abs(auc.mean() - expected_auc.mean()) < 0.01
NUM_EPOCHS = 3 ITEM_ALPHA = 1e-6 # Let's fit a WARP model: these generally have the best performance. model = LightFM(loss='warp', item_alpha=ITEM_ALPHA, no_components=NUM_COMPONENTS) # Run 3 epochs and time it. model = model.fit(train, epochs=NUM_EPOCHS, num_threads=NUM_THREADS) # Import the evaluation routines from lightfm.evaluation import auc_score # Compute and print the AUC score train_auc = auc_score(model, train, num_threads=NUM_THREADS).mean() print('Collaborative filtering train AUC: %s' % train_auc) # We pass in the train interactions to exclude them from predictions. # This is to simulate a recommender system where we do not # re-recommend things the user has already interacted with in the train # set. test_auc = auc_score(model, test, train_interactions=train, num_threads=NUM_THREADS).mean() print('Collaborative filtering test AUC: %s' % test_auc) # Set biases to zero model.item_biases *= 0.0 test_auc = auc_score(model, test, train_interactions=train, num_threads=NUM_THREADS).mean()
def do_fiber_training(visualization = False): if not os.path.isfile(rc.RECOMMENDER_TRAINING) or not os.path.isfile(rc.RECOMMENDER_MODEL): yarn_data_matrix = pickle.load(open( rc.YARN_DATA_MATRIX, "rb" )) yarn_data_train = sps.coo_matrix( yarn_data_matrix[:int(len(yarn_data_matrix)*0.5)] ) > 0 yarn_data_test = sps.coo_matrix( yarn_data_matrix[int(len(yarn_data_matrix)*0.5):] ) > 0 if visualization: print yarn_data_train.shape[0],yarn_data_test.shape[0], len(yarn_data_matrix) # Taken from: https://github.com/lyst/lightfm/blob/master/examples/stackexchange/hybrid_crossvalidated.ipynb # Set the number of threads; you can increase this # ify you have more physical cores available. NUM_THREADS = 2 NUM_COMPONENTS = 30 NUM_EPOCHS = 3 ITEM_ALPHA = 1e-6 # Let's fit a WARP model: these generally have the best performance. model = LightFM(loss='warp', item_alpha=ITEM_ALPHA, no_components=NUM_COMPONENTS) # Run 3 epochs and time it. model = model.fit(yarn_data_train, epochs=NUM_EPOCHS, num_threads=NUM_THREADS) # Compute and print the AUC score train_auc = auc_score(model, yarn_data_train, num_threads=NUM_THREADS).mean() print('Collaborative filtering train AUC: %s' % train_auc) # We pass in the train interactions to exclude them from predictions. # This is to simulate a recommender system where we do not # re-recommend things the user has already interacted with in the train # set. test_auc = auc_score(model, yarn_data_test, train_interactions=yarn_data_train, num_threads=NUM_THREADS).mean() print('Collaborative filtering test AUC: %s' % test_auc) pickle.dump(yarn_data_matrix,open(rc.RECOMMENDER_TRAINING, 'wb')) pickle.dump(model,open(rc.RECOMMENDER_MODEL, 'wb')) else: yarn_data_matrix = pickle.load(open(rc.RECOMMENDER_TRAINING, 'rb')) model = pickle.load(open(rc.RECOMMENDER_MODEL, 'rb')) translation_dict = pickle.load(open(rc.YARN_TRANSLATION_DATA, 'rb')) print len(yarn_data_matrix) for matrix_id in xrange(0,len(yarn_data_matrix)): print matrix_id predictions = model.predict(matrix_id,yarn_data_matrix[matrix_id]) matches = [] predictions += abs(np.min(predictions)) # make non-negative _max = np.max(predictions) # find max for normalization predictions /= _max # Normalize predictions for prediction in xrange(0,len(predictions)): if predictions[prediction] > 0.9: matches.append([translation_dict[prediction],prediction,predictions[prediction]]) print translation_dict[matrix_id],matches