def calculate_metrics(df): """ Calculates metrics at different k (1 to 10 + 20,30,40,50)""" #print(df.columns) klist = list(range(1, 11)) klist.extend([20, 30, 40, 50, 100, 200, 300, 500]) print(klist) # 14 x 3 x 4 columns added for each for k in tqdm(klist): df['average_precision_p2v_{}'.format(k)] = df['p2v_binary'].progress_apply(lambda x: average_precision(x, k)) df['average_precision_d2v_{}'.format(k)] = df['d2v_binary'].progress_apply(lambda x: average_precision(x, k)) df['recall_p2v_{}'.format(k)] = df[['p2v_binary', 'ground_truth']].apply( lambda x: recall_at_k(x.p2v_binary, x.ground_truth, k), axis=1) df['recall_d2v_{}'.format(k)] = df[['d2v_binary', 'ground_truth']].progress_apply( lambda x: recall_at_k(x.d2v_binary, x.ground_truth, k), axis=1) df['reciprocal_rank_p2v_{}'.format(k)] = df['p2v_binary'].progress_apply(lambda x: reciprocal_rank(x, k)) df['reciprocal_rank_d2v_{}'.format(k)] = df['d2v_binary'].progress_apply(lambda x: reciprocal_rank(x, k)) df['ndcg_p2v_{}'.format(k)] = df['p2v_binary'].progress_apply(lambda x: ndcg(x, k)) df['ndcg_d2v_{}'.format(k)] = df['d2v_binary'].progress_apply(lambda x: ndcg(x, k)) #df.to_csv('/home/ashwath/Programs/UnpaywallMAG/Evaluation/paper2vec_unpaywall_500.tsv', sep='\t') df.to_pickle('/home/ashwath/Programs/UnpaywallMAG/Pickles/paperwisemetrics_unpaywall_p2v_d2v_df_may21.pickle') print("METRICS CALCULATED, time to calculate the means") # Get the mean of all the index columns df = df.drop(['p2v_recommendations', 'p2v_binary', 'd2v_recommendations', 'd2v_binary', 'ground_truth'], axis=1) mean_series = df.mean() mean_series.to_csv('/home/ashwath/Programs/UnpaywallMAG/Evaluation/meanmetrics_p2v_d2v_may21.tsv', sep='\t', index=True, header=False) print("C'est fini.")
def dev_step(): acc = [] losses = [] pred_scores = [] ture_scores = [] count = 0 while True: try: feed_dict = { handle: test_handle, model.dropout_keep_prob: 1.0 } step, loss, accuracy, y_pred, target = sess.run([ global_step, model.loss, model.accuracy, model.y_pred, model.target ], feed_dict) acc.append(accuracy) losses.append(loss) pred_scores += list(y_pred[:, 1]) ture_scores += list(target) count += 1 if count % 1000 == 0: print("Processing {} samples".format(count)) except tf.errors.OutOfRangeError: break MeanAcc = sum(acc) / len(acc) MeanLoss = sum(losses) / len(losses) with open( os.path.join(out_dir, 'predScores-iter-%s-test.txt' % (step)), 'w') as f: for score1, score2 in zip(pred_scores, ture_scores): f.writelines(str(score1) + '\t' + str(score2) + '\n') num_sample = int(len(pred_scores) / 10) score_list = np.split(np.array(pred_scores), num_sample, axis=0) recall_2_1 = recall_2at1(score_list, k=1) recall_at_1 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 1) recall_at_2 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 2) recall_at_5 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 5) time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print("**********************************") print('pred_scores: ', len(pred_scores)) print("recall_2_1: %.3f" % (recall_2_1)) print("recall_at_1: %.3f" % (recall_at_1)) print("recall_at_2: %.3f" % (recall_at_2)) print("recall_at_5: %.3f" % (recall_at_5)) print("**********************************")
def dev_step(): acc = [] losses = [] pred_scores = [] ture_scores = [] count = 0 while True: try: feed_dict = { handle: test_handle, model.is_training: False, model.dropout_keep_prob: 1.0 } step, loss, accuracy, y_pred, target = sess.run([ global_step, model.loss, model.accuracy, model.y_pred, model.target ], feed_dict) acc.append(accuracy) losses.append(loss) pred_scores += list(y_pred[:, 1]) ture_scores += list(target) count += 1 if count % 1000 == 0: print(count) except tf.errors.OutOfRangeError: break MeanAcc = sum(acc) / len(acc) MeanLoss = sum(losses) / len(losses) if ('ubuntu' in FLAGS.data_path): num_sample = int(len(pred_scores) / 10) score_list = np.split(np.array(pred_scores), num_sample, axis=0) recall_2_1 = recall_2at1(score_list, k=1) recall_at_1 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 1) recall_at_2 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 2) recall_at_5 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 5) time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print("**********************************") print("%s results.........." % (flag.title())) print('pred_scores: ', len(pred_scores)) print("Step: %d \t| loss: %.3f \t| acc: %.3f \t| %s" % (step, MeanLoss, MeanAcc, time_str)) print("recall_2_1: %.3f" % (recall_2_1)) print("recall_at_1: %.3f" % (recall_at_1)) print("recall_at_2: %.3f" % (recall_at_2)) print("recall_at_5: %.3f" % (recall_at_5)) print("**********************************")
def topk_eval(sess, args, user_path_top_k, model, user_list, train_record, eval_record, test_record, item_set, k_list, batch_size, mode = 'test'): precision_list = {k: [] for k in k_list} recall_list = {k: [] for k in k_list} MAP_list = {k: [] for k in k_list} hit_ratio_list = {k: [] for k in k_list} ndcg_list = {k: [] for k in k_list} for user in user_list: if mode == 'eval': ref_user = eval_record else: ref_user = test_record if user in ref_user: test_item_list = list(item_set - train_record[user]) # test_item_list = test_item_list[:1000] item_score_map = dict() start = 0 while start + batch_size <= len(test_item_list): items, scores = model.get_scores(sess, {model.user_indices: [user] * batch_size, model.item_indices: test_item_list[start:start + batch_size]}) for item, score in zip(items, scores): item_score_map[item] = score start += batch_size # padding the last incomplete minibatch if exists if start < len(test_item_list): items, scores = model.get_scores( sess, {model.user_indices: [user] * batch_size, model.item_indices: test_item_list[start:] + [test_item_list[-1]] * ( batch_size - len(test_item_list) + start)}) for item, score in zip(items, scores): item_score_map[item] = score item_score_pair_sorted = sorted(item_score_map.items(), key=lambda x: x[1], reverse=True) item_sorted = [i[0] for i in item_score_pair_sorted] for k in k_list: recall_list[k].append(recall_at_k(item_sorted,ref_user[user],k)) # ndcg r_hit = [] for i in item_sorted[:k]: if i in ref_user[user]: r_hit.append(1) else: r_hit.append(0) for k in k_list: ndcg_list[k].append(ndcg_at_k(r_hit,k)) precision = [np.mean(precision_list[k]) for k in k_list] recall = [np.mean(recall_list[k]) for k in k_list] ndcg = [np.mean(ndcg_list[k]) for k in k_list] # MAP = [np.mean(MAP_list[k]) for k in k_list] # hit_ratio = [np.mean(hit_ratio_list[k]) for k in k_list] return precision, recall, ndcg, None, None
def metric_test(self): """ uses tensorrec eval as benchmark for rating performance of various reco algorithms """ k = 10 latent_factor = 10 n_users = 10 n_items = 12 interactions, user_features, item_features = util.generate_dummy_data_with_indicator( num_users=n_users, num_items=n_items, interaction_density=.5) print("interactiosn shape={}".format(np.shape(interactions))) print("user features shape={}".format(np.shape( user_features.toarray()))) print("item features shape={}".format(np.shape( item_features.toarray()))) model = TensorRec(n_components=latent_factor) model.fit(interactions, user_features, item_features, epochs=19) ranks = model.predict_rank(user_features=user_features, item_features=item_features) print("Ranks shape={}".format(np.shape(ranks))) self.assertTrue(np.shape(interactions) == np.shape(ranks)) tr_recall_result = eval.recall_at_k(predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print (tr_recall_result.mean()) tr_precision_result = eval.precision_at_k( predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print(tr_precision_result.mean()) # we need csr for interactions data interactions_ = interactions.tocsr() recall_result = metrics.recall_at_k(ranks, interactions_, k=k, preserve_rows=False) # print(recall_result.mean()) precision_result = metrics.precision_at_k(ranks, interactions_, k=k, preserve_rows=False) # print (precision_result.mean()) self.assertTrue(tr_recall_result.mean() == recall_result.mean()) self.assertTrue(tr_precision_result.mean() == precision_result.mean())
def get_performance(user_pos_test, r, auc, Ks): precision, recall, ndcg, hit_ratio = [], [], [], [] for K in Ks: precision.append(metrics.precision_at_k(r, K)) recall.append(metrics.recall_at_k(r, K, len(user_pos_test))) ndcg.append(metrics.ndcg_at_k(r, K)) hit_ratio.append(metrics.hit_at_k(r, K)) return { 'recall': np.array(recall), 'precision': np.array(precision), 'ndcg': np.array(ndcg), 'hit_ratio': np.array(hit_ratio), 'auc': auc }
def dev_step(flag, writer): sess.run(valid_iterator.initializer) valid_handle = sess.run(valid_iterator.string_handle()) acc = [] losses = [] pred_scores = [] ture_scores = [] count = 0 while True: try: feed_dict = { handle: valid_handle, model.is_training: False, model.dropout_keep_prob: 1.0 } step, loss, accuracy, y_pred, target = sess.run([ global_step, model.loss, model.accuracy, model.y_pred, model.target ], feed_dict) acc.append(accuracy) losses.append(loss) pred_scores += list(y_pred[:, 1]) ture_scores += list(target) count += 1 if count % 1000 == 0: print(count) except tf.errors.OutOfRangeError: break MeanAcc = sum(acc) / len(acc) MeanLoss = sum(losses) / len(losses) with open( os.path.join(out_dir, 'predScores-iter-%s.txt' % (step)), 'w') as f: for score1, score2 in zip(pred_scores, ture_scores): f.writelines(str(score1) + '\t' + str(score2) + '\n') summary_MeanLoss = tf.Summary(value=[ tf.Summary.Value(tag='%s/MeanLoss' % (flag), simple_value=MeanLoss) ]) summary_MeanAcc = tf.Summary(value=[ tf.Summary.Value(tag='%s/MeanAcc' % (flag), simple_value=MeanAcc) ]) writer.add_summary(summary_MeanLoss, step) writer.add_summary(summary_MeanAcc, step) if ('ubuntu' in FLAGS.data_path): num_sample = int(len(pred_scores) / 10) score_list = np.split(np.array(pred_scores), num_sample, axis=0) recall_2_1 = recall_2at1(score_list, k=1) recall_at_1 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 1) recall_at_2 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 2) recall_at_5 = recall_at_k(np.array(ture_scores), np.array(pred_scores), 5) time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S') print("**********************************") print("%s results.........." % (flag.title())) print('pred_scores: ', len(pred_scores)) print("Step: %d \t| loss: %.3f \t| acc: %.3f \t| %s" % (step, MeanLoss, MeanAcc, time_str)) print("recall_2_1: %.3f" % (recall_2_1)) print("recall_at_1: %.3f" % (recall_at_1)) print("recall_at_2: %.3f" % (recall_at_2)) print("recall_at_5: %.3f" % (recall_at_5)) print("**********************************") summary_recall_2_1 = tf.Summary(value=[ tf.Summary.Value(tag='%s/recall_2_1' % (flag), simple_value=recall_2_1) ]) summary_recall_at_1 = tf.Summary(value=[ tf.Summary.Value(tag='%s/recall_at_1' % (flag), simple_value=recall_at_1) ]) summary_recall_at_2 = tf.Summary(value=[ tf.Summary.Value(tag='%s/recall_at_2' % (flag), simple_value=recall_at_2) ]) summary_recall_at_5 = tf.Summary(value=[ tf.Summary.Value(tag='%s/recall_at_5' % (flag), simple_value=recall_at_5) ]) writer.add_summary(summary_recall_2_1, step) writer.add_summary(summary_recall_at_1, step) writer.add_summary(summary_recall_at_2, step) writer.add_summary(summary_recall_at_5, step) return MeanLoss, recall_2_1 + recall_at_1
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to n_user x n_item matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Take the mean only from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean:%.3f" % rating_mean) """ Find PQ sub matrices """ R = ratings.values """ Randomly initialize P & Q matrices with n latent factors """ n_factors = 10 P = np.random.normal(0, .1, (n_users, n_factors)) Q = np.random.normal(0, .1, (n_factors, n_items)) R_mask = R.copy() R_mask[R_mask != 0.000000] = 1 R_hat = np.zeros(np.shape(R)) R_hat_mask = np.zeros(np.shape(R)) np.matmul(P, Q, out=R_hat) # Get errors only from explicitly rated elements np.multiply(R_hat, R_mask, out=R_hat_mask) """ Compute error: MSE = (1/N) * (R - R_hat), RMSE = MSE^(1/2) """ diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), n_users * n_items) rmse = np.sqrt(mse) print("RMSE: %.5f" % (rmse)) print("Type: ", type(R_hat)) print(R_hat[:5, :10]) predicted_ranks = metric.rank_matrix(R_hat) print(predicted_ranks.shape) print(predicted_ranks[:5, :10]) ratings_csr = sparse.csr_matrix(ratings.values) precision = metrics.precision_at_k(predicted_ranks, ratings_csr, k=100) recall = metrics.recall_at_k(predicted_ranks, ratings_csr, k=100) print("Precision {0:.5f}% Recall={1:.5f}%".format(precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) normalization_enabled = False optimize_enabled = True k = 100 """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to user x movie matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) """ Construct data """ users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Compute mean ratingonly from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean: %.6f" % rating_mean) R_mask = np.zeros(np.shape(ratings)) R_mask[ratings != 0.000000] = 1 if normalization_enabled: temp = ratings.copy() ratings_norm = np.subtract(temp, rating_mean, where=temp != 0) ratings_norm = np.multiply(ratings_norm, R_mask) assert (np.count_nonzero(ratings_norm) == np.count_nonzero(ratings)) R = ratings_norm.values else: R = ratings.values.copy() # Setup covariance to treat the item columns as input variables covar = np.cov(R, rowvar=False) evals, evecs = np.linalg.eigh(covar) print("cov_mat shape: %s" % str(np.shape(covar))) print("evals shape: %s" % str(np.shape(evals))) print("evecs shape: %s" % str(np.shape(evecs))) n_components = 10 # principal components """ Randomly initialize weights table """ weights = np.random.normal(0, .1, (n_users, n_components)) components = evecs[:n_components, :n_items] R_hat_mask = np.zeros(np.shape(R), dtype=np.float64) if optimize_enabled: # optimization parameters epochs = 5 learning_rate = .0001 lambda_ = .0001 verbosity = 1 print("Optimized PCA epochs=%s" % epochs) """ We only modify the weight matrix """ for epoch in range(epochs): for u in range(n_users): for i in range(n_items): error = R[u, i] - np.dot(weights[u, :], components[:, i]) for k in range(n_components): weights[u, k] = weights[u, k] - learning_rate * ( error * -2 * components[k, i] + lambda_ * (2 * np.abs(weights[u, k]) + 2 * np.abs(components[k, i]))) R_hat = np.zeros(np.shape(R)) np.matmul(weights, components, out=R_hat) # Get errors only from explicitly rated elements np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), np.count_nonzero(R)) rmse = np.sqrt(mse) if epoch % verbosity == 0 or epoch == (epochs - 1): print("Epoch %d: RMSE: %.6f" % (epoch, rmse)) else: R_hat = np.matmul(weights, components) print("R_hat shape: %s" % str(np.shape(R_hat))) assert (np.shape(R) == np.shape(R_hat)) print("PCA single run") np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), np.count_nonzero(R)) rmse = np.sqrt(mse) print("RMSE: %.5f" % rmse) assert (R.shape == R_hat.shape) sparse_data = sparse.csr_matrix(R) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, sparse_data, k=k) recall = metrics.recall_at_k(predicted_ranks, sparse_data, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStoppping '%s" % sys.argv[0])
def deepfm_test(self): train_x, train_y = DeepFM.df2xy(self._ratings) #test_x, test_y = DeepFM.df2xy(self.test_data_) params = { 'n_uid': self._ratings.userid.max(), 'n_mid': self._ratings.itemid.max(), # 'n_genre': self.n_genre_, 'k': self._k, 'dnn_dim': [64, 64], 'dnn_dr': 0.5, 'filepath': '../data/deepfm_weights.h5' } """ train """ model = DeepFM(**params) train_history = model.fit(train_x, train_y, epochs=self._epochs, batch_size=2048, validation_split=0.1) history = pd.DataFrame(train_history.history) history.plot() plt.savefig("../data//history.png") """ test """ results = model.evaluate(train_x, train_y) print("Validate result:{0}".format(results)) """ predict """ y_hat = model.predict(train_x) print(np.shape(y_hat)) # print(np.shape(test_y)) """ Run Recall and Precision Metrics """ n_users = np.max(self._ratings.userid.values) + 1 n_items = np.max(self._ratings.itemid.values) + 1 print("n_users={0} n_items={1}".format(n_users, n_items)) # Convert to sparse matrix to run standard metrics sparse_train = sparse.coo_matrix((self._ratings.rating.values, (self._ratings.userid.values, self._ratings.itemid.values)), shape=(n_users, n_items)) # sparse_test = sparse.coo_matrix((self.test_data_.rating.values, \ # (self.test_data_.uid.values, self.test_data_.mid.values)), \ # shape=(n_users, n_items)) # pd.DataFrame(data=sparse_test.tocsr().todense().A).to_csv("./testdata.csv") # test_prediced test_predicted = self._ratings.copy() test_predicted.rating = np.round(y_hat) sparse_predicted = sparse.coo_matrix((test_predicted.rating.values, \ (test_predicted.userid.values, test_predicted.itemid.values)), \ shape=(n_users, n_items)) sparse_train_1up = sparse_train.multiply(sparse_train >= 1) # sparse_test_1up = sparse_test.multiply(sparse_test >= 1) predicted_arr = sparse_predicted.tocsr().todense().A predicted_ranks = metrics.rank_matrix(predicted_arr) precision_ = metrics.precision_at_k(predicted_ranks, sparse_train, k=self._k) recall_ = metrics.recall_at_k(predicted_ranks, sparse_train, k=self._k) print("{0}.xdeepfm_test train precision={1:.4f}% recall={2:.4f}% @k={3}".format( __class__.__name__, precision_ * 100, recall_ * 100, self._k))
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) k = 100 normalization_enabled = False """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to user x movie matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) # train_size = 0.7 # train_row_size = int(len(ratings.index) * train_size) # train_col_size = int(len(ratings.columns) * train_size) # ratings = ratings.loc[:train_row_size, :train_col_size] users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) assert (np.max(users) == len(users)) assert (np.max(items) == len(items)) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Take the mean only from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean: %.2f" % rating_mean) if normalization_enabled: temp = ratings.copy() ratings_norm = np.subtract(temp, rating_mean, where=temp != 0) R = ratings_norm.values else: R = ratings.values U, S, V = linalg.svds(R, k=k) # print ("U: ", np.shape(U)) # print ("S: ", np.shape(S)) # print ("V: ", np.shape(V)) sigma = np.diag(S) # print ("Sigma: ", np.shape(sigma)) """ Generate prediction matrix """ R_hat = np.dot(np.dot(U, sigma), V) assert (np.shape(R) == np.shape(R_hat)) # Get errors only from explicitly rated elements R_mask = np.zeros(np.shape(R)) R_mask[R != 0.000000] = 1 R_hat_mask = np.zeros(np.shape(R)) np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) assert (np.count_nonzero(R) == np.count_nonzero(R_hat_mask)) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) #mse = np.divide(diff_square.sum(), n_users*n_items) mse = np.divide(diff_square.sum(), np.count_nonzero(R_mask)) rmse = np.sqrt(mse) print("RMSE: %.6f" % (rmse)) assert (R.shape == R_hat.shape) interactions = sparse.csr_matrix(R) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, interactions, k=k) recall = metrics.recall_at_k(predicted_ranks, interactions, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])
def topk_eval(sess, args, data_generator, model, user_list, train_record, eval_record, test_record, item_set, k_list, batch_size, mode='test'): precision_list = {k: [] for k in k_list} recall_list = {k: [] for k in k_list} MAP_list = {k: [] for k in k_list} hit_ratio_list = {k: [] for k in k_list} ndcg_list = {k: [] for k in k_list} for user in user_list: if mode == 'eval': ref_user = eval_record else: ref_user = test_record if user in ref_user: test_item_list = list(item_set - train_record[user]) item_score_map = dict() start = 0 while start + batch_size <= len(test_item_list): user_list_tmp = [user] * batch_size user_list_tmp = np.array(user_list_tmp) item_list = test_item_list[start:start + batch_size] item_list = np.array(item_list) labels_list = [1] * batch_size labels_list = np.array(labels_list) data = np.concatenate((np.expand_dims( user_list_tmp, axis=1), np.expand_dims(item_list, axis=1), np.expand_dims(labels_list, axis=1)), axis=1) batch_data = data_generator.generate_rating_batch( data, 0, args.batch_size) feed_dict = data_generator.generate_feed_rating_dict( model, batch_data) items, scores = model.get_scores(sess, feed_dict) for item, score in zip(items, scores): item_score_map[item] = score start += batch_size # padding the last incomplete minibatch if exists if start < len(test_item_list): user_list_tmp = [user] * batch_size user_list_tmp = np.array(user_list_tmp) item_list = test_item_list[start:] + [test_item_list[-1]] * ( batch_size - len(test_item_list) + start) item_list = np.array(item_list) labels_list = [1] * batch_size labels_list = np.array(labels_list) data = np.concatenate((np.expand_dims( user_list_tmp, axis=1), np.expand_dims(item_list, axis=1), np.expand_dims(labels_list, axis=1)), axis=1) batch_data = data_generator.generate_rating_batch( data, 0, args.batch_size) feed_dict = data_generator.generate_feed_rating_dict( model, batch_data) items, scores = model.get_scores(sess, feed_dict) for item, score in zip(items, scores): item_score_map[item] = score item_score_pair_sorted = sorted(item_score_map.items(), key=lambda x: x[1], reverse=True) item_sorted = [i[0] for i in item_score_pair_sorted] for k in k_list: precision_list[k].append( precision_at_k(item_sorted, ref_user[user], k)) recall_list[k].append( recall_at_k(item_sorted, ref_user[user], k)) # ndcg r_hit = [] for i in item_sorted[:k]: if i in ref_user[user]: r_hit.append(1) else: r_hit.append(0) for k in k_list: ndcg_list[k].append(ndcg_at_k(r_hit, k)) precision = [np.mean(precision_list[k]) for k in k_list] recall = [np.mean(recall_list[k]) for k in k_list] ndcg = [np.mean(ndcg_list[k]) for k in k_list] return precision, recall, ndcg, None, None
def main(): session = tf.Session() normalized_on = False k = 100 """ load dataset """ datafile = "./data/ml-100k/u.data" df = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) n_users = len(np.unique(df.userid)) n_items = len(np.unique(df.itemid)) rating_mean = np.mean(df.rating) rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Raw data:") print("Shape: %s" % str(df.shape)) print("Userid size: %d" % n_users) print("Itemid size: %d" % n_items) print("Rating mean: %.5f" % rating_mean) """ Format ratings to user x item matrix """ df = df.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(df, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) print("Raw ratings size", len(ratings)) ratings = ratings.astype(np.float64) """ Construct training data """ # train_size = 0.7 ratings_train_ = ratings #.loc[:int(n_users*train_size), :int(n_items*train_size)] users = ratings_train_.index.values items = ratings_train_.columns.values n_users = len(users) n_items = len(items) temp = ratings_train_.copy() rating_mean = temp.replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Training data:") print("Shape: %s" % str(ratings_train_.shape)) print("n_users: %d" % n_users) print("n_items: %d" % n_items) print("rating mean: %.5f" % rating_mean) user_indices = [x for x in range(n_users)] item_indices = [x for x in range(n_items)] print("Max userid train: ", np.max(users)) print("Max itemid train", np.max(items)) print("user_indices size ", len(user_indices)) print("item_indices size ", len(item_indices)) if normalized_on: ratings_norm = np.zeros(ratings_train_.shape) temp = ratings_train_.values np.subtract(temp, rating_mean, where=temp != 0, out=ratings_norm) ratings = ratings_norm else: ratings = ratings_train_.values # Variables n_features = 10 # latent factors U = tf.Variable(initial_value=tf.truncated_normal([n_users, n_features])) P = tf.Variable(initial_value=tf.truncated_normal([n_features, n_items])) result = tf.matmul(U, P) result_flatten = tf.reshape(result, [-1]) assert (result_flatten.shape[0] == n_users * n_items) R = tf.gather(result_flatten, user_indices[:-1] * n_items + item_indices) assert (R.shape[0] == n_users * n_items) R_ = tf.reshape(R, [tf.div(R.shape[0], n_items), len(item_indices)]) assert (R_.shape == ratings.shape) """ Compute error for values from the original ratings matrix so that means excluding values implicitly computed by UxP """ var = tf.Variable(ratings.astype(np.float32)) compare = tf.not_equal(var, tf.constant(0.0)) compare_op = var.assign(tf.where(compare, tf.ones_like(var), var)) R_masked = tf.multiply(R_, compare_op) assert (ratings.shape == R_masked.shape) """ Cost function: sum_ij{ |r_ij- rhat_ij| + lambda*(|u_i|+|p_j|)} """ diff_op = tf.subtract(ratings.astype(np.float32), R_masked) diff_op_abs = tf.abs(diff_op) base_cost = tf.reduce_sum(diff_op_abs) # Regularizer sum_ij{lambda*(|U_i| + |P_j|)} lambda_ = tf.constant(.001) norm_sums = tf.add(tf.reduce_sum(tf.abs(U)), tf.reduce_sum(tf.abs(P))) regularizer = tf.multiply(norm_sums, lambda_) cost = tf.add(base_cost, regularizer) """ Optimizer """ lr = tf.constant(.0001) global_step = tf.Variable(0, trainable=False) decaying_learning_rate = tf.train.exponential_decay(lr, global_step, 10000, .96, staircase=True) optimizer = tf.train.GradientDescentOptimizer( decaying_learning_rate).minimize(cost, global_step=global_step) """ Run """ init = tf.global_variables_initializer() session.run(init) print("Running stochastic gradient descent..") epoch = 500 for i in range(epoch): session.run(optimizer) if i % 10 == 0 or i == epoch - 1: diff_op_train = tf.subtract(ratings.astype(np.float32), R_masked) diff_op_train_squared = tf.square(diff_op_train) se = tf.reduce_sum(diff_op_train_squared) mse = tf.divide(se, n_users * n_items) rmse = tf.sqrt(mse) print("Train iter: %d MSE: %.5f loss: %.5f" % (i, session.run(rmse), session.run(cost))) R_hat = R_.eval(session=session) predicted_ranks = metrics.rank_matrix(R_hat) interactions = sparse.csr_matrix(ratings) precision = metrics.precision_at_k(predicted_ranks, interactions, k=k) recall = metrics.recall_at_k(predicted_ranks, interactions, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100))
def main(): logging.info('reading data') item_mat = data.get_mult() trainM = sparse.csr_matrix( data.read_user(f_in='data/dummy/cf-train-10-users.dat', num_u=50, num_v=1929)) testM = sparse.csr_matrix( data.read_user(f_in='data/dummy/cf-test-10-users.dat', num_u=50, num_v=1929)) trainList = list() testList = list() for user in range(trainM.shape[0]): negative = 0 for item in range(trainM.shape[1]): if trainM[user, item] == 1: trainList.append([user, item, 1]) else: if negative < 20: trainList.append([user, item, 0]) negative += 1 train = np.array(trainList).astype('float32') testList = list() for user in range(testM.shape[0]): negative = 0 for item in range(testM.shape[1]): if testM[user, item] == 1: testList.append([user, item, 1]) # else: # if negative < 10: # testList.append( [user, item, 0] ) # negative+=1 test = np.array(testList).astype('float32') num_item_feat = item_mat.shape[1] model = CollaborativeDeepLearning(item_mat, [num_item_feat, 50, 10]) model.pretrain(lamda_w=0.001, encoder_noise=0.3, epochs=10) model_history = model.fineture(train, test, lamda_u=0.01, lamda_v=0.1, lamda_n=0.1, lr=0.01, epochs=500) testing_rmse = model.getRMSE(test) print('Testing RMSE = {}'.format(testing_rmse)) import metrics print('AUC %s' % metrics.full_auc(model.cdl_model, testM)) import matplotlib.pyplot as plt M_low = 50 M_high = 300 recall_levels = M_high - M_low + 1 recallArray = np.zeros(6) x = 0 for n in [50, 100, 150, 200, 250, 300]: test_recall = metrics.recall_at_k(model.cdl_model, testM, k=n) recallArray[x] = test_recall print('Recall: %.2f.' % (test_recall)) x += 1 plt.plot([50, 100, 150, 200, 250, 300], recallArray) plt.ylabel("Recall") plt.xlabel("M") plt.title("Proposed: Recall@M") plt.show()
def main(): print("\nStarting '%s'" % sys.argv[0]) session = tf.Session() normalized_on = True """ load dataset """ datafile = "./data/ml-100k/u.data" df = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) n_users = len(np.unique(df.userid)) n_items = len(np.unique(df.itemid)) rating_mean = np.mean(df.rating) rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print ("Raw data:") print ("Shape: %s" % str(df.shape)) print ("Userid size: %d" % n_users) print ("Itemid size: %d" % n_items) print ("Rating mean: %.5f" % rating_mean) """ Format ratings to user x item matrix """ df = df.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(df, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) print("Raw ratings size", len(ratings)) ratings = ratings.astype(np.float64) """ Construct training data """ # train_factor = 0.7 # train_size = int(n_users*train_factor) # ratings_train_ = ratings.loc[:train_size, :int(n_items*train_factor)] users = ratings.index.values items = ratings.columns.values n_users = len(users) n_items = len(items) temp = ratings.copy() rating_mean = temp.replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print ("Training data:") print ("Shape: %s" % str(ratings.shape)) print ("n_users: %d" % n_users) print ("n_items: %d" % n_items) print ("rating mean: %.5f" % rating_mean) user_indices = [x for x in range(n_users)] item_indices = [x for x in range(n_items)] print ("Max userid train: %d" % np.max(users)) print ("Max itemid train: %d" % np.max(items)) print ("user_indices size: %d" % len(user_indices)) print ("item_indices size: %d " % len(item_indices)) if normalized_on: ratings_norm = np.zeros(ratings.shape) temp = ratings.values np.subtract(temp, rating_mean, where=temp!=0, out=ratings_norm) ratings = ratings_norm else: ratings = ratings.values # Variables n_features = 10 # latent factors U = tf.Variable(initial_value=tf.truncated_normal([n_users, n_features])) P = tf.Variable(initial_value=tf.truncated_normal([n_features, n_items])) result = tf.matmul(U, P) result_flatten = tf.reshape(result, [-1]) assert (result_flatten.shape[0] == n_users * n_items) print ("user indices size: %d item indices size: %d" % (len(user_indices), len(item_indices))) # Fill R from result_flatten R = tf.gather(result_flatten, user_indices[:-1] * n_items + item_indices) assert (R.shape == result_flatten.shape) # Format R to user x item sized matrix R_ = tf.reshape(R, [tf.div(R.shape[0], n_items), len(item_indices)]) assert (R_.shape == ratings.shape) """ Compute error of fields from the original ratings matrix """ var = tf.Variable(ratings.astype(np.float32)) compare = tf.not_equal(var, tf.constant(0.0)) compare_op = var.assign(tf.where(compare, tf.ones_like(var), var)) R_mask = tf.multiply(R_, compare_op) assert (R_mask.shape == np.shape(ratings)) """ Cost function: sum_ij{ |r_ij- rhat_ij| + lambda*(|u_i|+|p_j|)} """ # cost |r - r_hat| diff_op = tf.subtract(ratings.astype(np.float32), R_mask) diff_op_abs = tf.abs(diff_op) base_cost = tf.reduce_sum(diff_op_abs) lambda_ = tf.constant(.001) norm_sums = tf.add(tf.reduce_sum(tf.abs(U)), tf.reduce_sum(tf.abs(P))) regularizer = tf.multiply(norm_sums, lambda_) cost = tf.add(base_cost, regularizer) """ Run """ init = tf.global_variables_initializer() session.run(init) session.run(cost) """ Mean square error """ diff_op_train = tf.subtract(ratings.astype(np.float32), R_mask) diff_op_train_squared = tf.square(diff_op_train) diff_op = tf.sqrt(tf.reduce_sum(diff_op_train_squared)) cost_train = tf.divide(diff_op, ratings.shape[0]) cost_train_result = session.run(cost_train) print("Training MSE: %.5f" % cost_train_result) k = 100 R_hat = R_.eval(session=session) print (ratings[:5, :5]) print (R_hat[:5,:5]) interactions = sparse.csr_matrix(ratings) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, interactions, k=100) recall = metrics.recall_at_k(predicted_ranks, interactions, k=100) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])