def evaluation(pred_scores, true_scores, samples=10): ''' :param pred_scores: list of scores predicted by model :param true_scores: list of ground truth labels, 1 or 0 :return: ''' num_sample = int(len(pred_scores) / samples) # 1 positive and 9 negative score_list = np.split(np.array(pred_scores), num_sample, axis=0) recall_2_1 = recall_2at1(np.array(true_scores), np.array(pred_scores)) recall_at_1 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 1) recall_at_2 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 2) recall_at_5 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 5) _mrr = MRR(np.array(true_scores), np.array(pred_scores)) _map = MAP(np.array(true_scores), np.array(pred_scores)) precision_at_1 = precision_at_k(np.array(true_scores), np.array(pred_scores), k=1) return { 'MAP': _map, 'MRR': _mrr, 'p@1': precision_at_1, 'r2@1': recall_2_1, 'r@1': recall_at_1, 'r@2': recall_at_2, 'r@5': recall_at_5, }
def score(sk_labels, im_labels, index): res = np.equal(im_labels[index], sk_labels[:, None]) prec = np.mean([precision_at_k(r, 100) for r in res]) pool = mp.Pool(processes=10) results = [pool.apply_async(average_precision, args=(r, )) for r in res] mAP = np.mean([p.get() for p in results]) pool.close() return prec, mAP
def metric_test(self): """ uses tensorrec eval as benchmark for rating performance of various reco algorithms """ k = 10 latent_factor = 10 n_users = 10 n_items = 12 interactions, user_features, item_features = util.generate_dummy_data_with_indicator( num_users=n_users, num_items=n_items, interaction_density=.5) print("interactiosn shape={}".format(np.shape(interactions))) print("user features shape={}".format(np.shape( user_features.toarray()))) print("item features shape={}".format(np.shape( item_features.toarray()))) model = TensorRec(n_components=latent_factor) model.fit(interactions, user_features, item_features, epochs=19) ranks = model.predict_rank(user_features=user_features, item_features=item_features) print("Ranks shape={}".format(np.shape(ranks))) self.assertTrue(np.shape(interactions) == np.shape(ranks)) tr_recall_result = eval.recall_at_k(predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print (tr_recall_result.mean()) tr_precision_result = eval.precision_at_k( predicted_ranks=ranks, test_interactions=interactions, k=k, preserve_rows=False) # print(tr_precision_result.mean()) # we need csr for interactions data interactions_ = interactions.tocsr() recall_result = metrics.recall_at_k(ranks, interactions_, k=k, preserve_rows=False) # print(recall_result.mean()) precision_result = metrics.precision_at_k(ranks, interactions_, k=k, preserve_rows=False) # print (precision_result.mean()) self.assertTrue(tr_recall_result.mean() == recall_result.mean()) self.assertTrue(tr_precision_result.mean() == precision_result.mean())
def get_performance(user_pos_test, r, auc, Ks): precision, recall, ndcg, hit_ratio = [], [], [], [] for K in Ks: precision.append(metrics.precision_at_k(r, K)) recall.append(metrics.recall_at_k(r, K, len(user_pos_test))) ndcg.append(metrics.ndcg_at_k(r, K)) hit_ratio.append(metrics.hit_at_k(r, K)) return { 'recall': np.array(recall), 'precision': np.array(precision), 'ndcg': np.array(ndcg), 'hit_ratio': np.array(hit_ratio), 'auc': auc }
def score_shape(sk_labels, im_labels, index): vv, cc = np.unique(im_labels, return_counts=True) lut = {} for v, c in zip(vv, cc): lut[v] = c res = np.equal(im_labels[index], sk_labels[:, None]) # 1-NN nn = np.mean(res[:, 0]) # first and second tier ft = np.mean( [np.sum(r[:lut[l]]) / float(lut[l]) for r, l in zip(res, sk_labels)]) st = np.mean([ np.sum(r[:2 * lut[l]]) / float(lut[l]) for r, l in zip(res, sk_labels) ]) # e-measure prec = np.mean([precision_at_k(r, 32) for r in res]) rec = np.mean( [np.sum(r[:32]) / float(lut[l]) for r, l in zip(res, sk_labels)]) e_measure = 2 * prec * rec / (prec + rec) # dcg pool = mp.Pool(processes=10) results = [ pool.apply_async(dcg_at_k, args=(r, len(im_labels), 1)) for r in res ] mDCG = np.mean([p.get() for p in results]) pool.close() # ndgc pool = mp.Pool(processes=10) results = [ pool.apply_async(ndcg_at_k, args=(r, len(im_labels), 1)) for r in res ] mnDCG = np.mean([p.get() for p in results]) pool.close() # map pool = mp.Pool(processes=10) results = [pool.apply_async(average_precision, args=(r, )) for r in res] mAP = np.mean([p.get() for p in results]) pool.close() return nn, ft, st, e_measure, mnDCG, mAP
def evaluation(pred_scores, true_scores, samples=10): ''' :param pred_scores: list of scores predicted by model :param true_scores: list of ground truth labels, 1 or 0 :return: ''' num_sample = int(len(pred_scores) / samples) # 1 positive and 9 negative # score_list = np.argmax(np.split(np.array(pred_scores), num_sample, axis=0), 1) # logit_list = np.argmax(np.split(np.array(true_scores), num_sample, axis=0), 1) recall_2_1 = recall_2at1(np.array(true_scores), np.array(pred_scores)) recall_at_1 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 1) recall_at_2 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 2) recall_at_5 = recall_at_k_new(np.array(true_scores), np.array(pred_scores), 5) _mrr = MRR(np.array(true_scores), np.array(pred_scores)) _map = MAP(np.array(true_scores), np.array(pred_scores)) precision_at_1 = precision_at_k(np.array(true_scores), np.array(pred_scores), k=1) # ndcg_at_1 = NDCG(np.array(true_scores), np.array(pred_scores), 1) # ndcg_at_2 = NDCG(np.array(true_scores), np.array(pred_scores), 2) # ndcg_at_5 = NDCG(np.array(true_scores), np.array(pred_scores), 5) print("**********************************") print("results..........") print('pred_scores: ', len(pred_scores)) print("MAP: %.3f" % (_map)) print("MRR: %.3f" % (_mrr)) print("precision_at_1: %.3f" % (precision_at_1)) print("recall_2_1: %.3f" % (recall_2_1)) print("recall_at_1: %.3f" % (recall_at_1)) print("recall_at_2: %.3f" % (recall_at_2)) print("recall_at_5: %.3f" % (recall_at_5)) print("**********************************") return { 'MAP': _map, 'MRR': _mrr, 'p@1': precision_at_1, 'r2@1': recall_2_1, 'r@1': recall_at_1, 'r@2': recall_at_2, 'r@5': recall_at_5, }
def evalMetric(self, metric): score, n = 0, 0 if self.verbose: print('Running evaluation') for w1 in self.eval: if w1 not in self.ranks: # skip non-existing continue # TODO: Implement desired metrics here if metric == 'precision_at_10': k = 10 rs = [1 if w in [x[0] for x in self.eval[w1]] else 0 for (w, _) in self.ranks[w1][:k]] # print w1 # print rs # print self.ranks[w1][:k] # print self.eval[w1] score += metrics.precision_at_k(rs, k) elif metric == 'map': rs = [1 if w in [x[0] for x in self.eval[w1]] else 0 for (w, _) in self.ranks[w1]] # print w1 # print rs # print self.ranks[w1] # print self.eval[w1] score += metrics.mean_average_precision(rs) elif metric == 'ndcg_at_100': k = 100 d = dict(self.eval[w1]) rs = [d[w] if w in d else 0 for (w, s) in self.ranks[w1][:k]] # print w1 # print rs # print self.ranks[w1][:k] # print self.eval[w1] score += metrics.ndcg_at_k(rs, k) n += 1 return (score / n, n)
def main(): print("\nStarting '%s'" % sys.argv[0]) session = tf.Session() normalized_on = True """ load dataset """ datafile = "./data/ml-100k/u.data" df = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) n_users = len(np.unique(df.userid)) n_items = len(np.unique(df.itemid)) rating_mean = np.mean(df.rating) rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print ("Raw data:") print ("Shape: %s" % str(df.shape)) print ("Userid size: %d" % n_users) print ("Itemid size: %d" % n_items) print ("Rating mean: %.5f" % rating_mean) """ Format ratings to user x item matrix """ df = df.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(df, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) print("Raw ratings size", len(ratings)) ratings = ratings.astype(np.float64) """ Construct training data """ # train_factor = 0.7 # train_size = int(n_users*train_factor) # ratings_train_ = ratings.loc[:train_size, :int(n_items*train_factor)] users = ratings.index.values items = ratings.columns.values n_users = len(users) n_items = len(items) temp = ratings.copy() rating_mean = temp.replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print ("Training data:") print ("Shape: %s" % str(ratings.shape)) print ("n_users: %d" % n_users) print ("n_items: %d" % n_items) print ("rating mean: %.5f" % rating_mean) user_indices = [x for x in range(n_users)] item_indices = [x for x in range(n_items)] print ("Max userid train: %d" % np.max(users)) print ("Max itemid train: %d" % np.max(items)) print ("user_indices size: %d" % len(user_indices)) print ("item_indices size: %d " % len(item_indices)) if normalized_on: ratings_norm = np.zeros(ratings.shape) temp = ratings.values np.subtract(temp, rating_mean, where=temp!=0, out=ratings_norm) ratings = ratings_norm else: ratings = ratings.values # Variables n_features = 10 # latent factors U = tf.Variable(initial_value=tf.truncated_normal([n_users, n_features])) P = tf.Variable(initial_value=tf.truncated_normal([n_features, n_items])) result = tf.matmul(U, P) result_flatten = tf.reshape(result, [-1]) assert (result_flatten.shape[0] == n_users * n_items) print ("user indices size: %d item indices size: %d" % (len(user_indices), len(item_indices))) # Fill R from result_flatten R = tf.gather(result_flatten, user_indices[:-1] * n_items + item_indices) assert (R.shape == result_flatten.shape) # Format R to user x item sized matrix R_ = tf.reshape(R, [tf.div(R.shape[0], n_items), len(item_indices)]) assert (R_.shape == ratings.shape) """ Compute error of fields from the original ratings matrix """ var = tf.Variable(ratings.astype(np.float32)) compare = tf.not_equal(var, tf.constant(0.0)) compare_op = var.assign(tf.where(compare, tf.ones_like(var), var)) R_mask = tf.multiply(R_, compare_op) assert (R_mask.shape == np.shape(ratings)) """ Cost function: sum_ij{ |r_ij- rhat_ij| + lambda*(|u_i|+|p_j|)} """ # cost |r - r_hat| diff_op = tf.subtract(ratings.astype(np.float32), R_mask) diff_op_abs = tf.abs(diff_op) base_cost = tf.reduce_sum(diff_op_abs) lambda_ = tf.constant(.001) norm_sums = tf.add(tf.reduce_sum(tf.abs(U)), tf.reduce_sum(tf.abs(P))) regularizer = tf.multiply(norm_sums, lambda_) cost = tf.add(base_cost, regularizer) """ Run """ init = tf.global_variables_initializer() session.run(init) session.run(cost) """ Mean square error """ diff_op_train = tf.subtract(ratings.astype(np.float32), R_mask) diff_op_train_squared = tf.square(diff_op_train) diff_op = tf.sqrt(tf.reduce_sum(diff_op_train_squared)) cost_train = tf.divide(diff_op, ratings.shape[0]) cost_train_result = session.run(cost_train) print("Training MSE: %.5f" % cost_train_result) k = 100 R_hat = R_.eval(session=session) print (ratings[:5, :5]) print (R_hat[:5,:5]) interactions = sparse.csr_matrix(ratings) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, interactions, k=100) recall = metrics.recall_at_k(predicted_ranks, interactions, k=100) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])
def score_single(sk_labels, im_labels, index): res = np.equal(im_labels[index], sk_labels[:, None]) prec = np.mean([precision_at_k(r, 100) for r in res]) mAP = mean_average_precision(res) return prec, mAP
def topk_eval(sess, args, user_triplet_set, model, user_list, train_record, eval_record, test_record, item_set, k_list, batch_size, mode='test'): precision_list = {k: [] for k in k_list} recall_list = {k: [] for k in k_list} MAP_list = {k: [] for k in k_list} hit_ratio_list = {k: [] for k in k_list} ndcg_list = {k: [] for k in k_list} for user in user_list: if mode == 'eval': ref_user = eval_record else: ref_user = test_record if user in ref_user: test_item_list = list(item_set - train_record[user]) item_score_map = dict() start = 0 while start + batch_size <= len(test_item_list): data = [] user_list_tmp = [user] * batch_size item_list = test_item_list[start:start + batch_size] labels_list = [1] * batch_size items, scores = model.get_scores( sess, get_feed_dict_top_k(args, model, user_list_tmp, item_list, labels_list, user_triplet_set)) for item, score in zip(items, scores): item_score_map[item] = score start += batch_size # padding the last incomplete minibatch if exists if start < len(test_item_list): user_list_tmp = [user] * batch_size item_list = test_item_list[start:] + [test_item_list[-1]] * ( batch_size - len(test_item_list) + start) labels_list = [1] * batch_size # items, scores = model.get_scores( # sess, {model.user_indices: [user] * batch_size, # model.item_indices: test_item_list[start:] + [test_item_list[-1]] * ( # batch_size - len(test_item_list) + start)}) items, scores = model.get_scores( sess, get_feed_dict_top_k(args, model, user_list_tmp, item_list, labels_list, user_triplet_set)) for item, score in zip(items, scores): item_score_map[item] = score item_score_pair_sorted = sorted(item_score_map.items(), key=lambda x: x[1], reverse=True) item_sorted = [i[0] for i in item_score_pair_sorted] for k in k_list: precision_list[k].append( precision_at_k(item_sorted, ref_user[user], k)) recall_list[k].append( recall_at_k(item_sorted, ref_user[user], k)) # ndcg r_hit = [] for i in item_sorted[:k]: if i in ref_user[user]: r_hit.append(1) else: r_hit.append(0) for k in k_list: ndcg_list[k].append(ndcg_at_k(r_hit, k)) precision = [np.mean(precision_list[k]) for k in k_list] recall = [np.mean(recall_list[k]) for k in k_list] ndcg = [np.mean(ndcg_list[k]) for k in k_list] return precision, recall, ndcg, None, None
def main(): session = tf.Session() normalized_on = False k = 100 """ load dataset """ datafile = "./data/ml-100k/u.data" df = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) n_users = len(np.unique(df.userid)) n_items = len(np.unique(df.itemid)) rating_mean = np.mean(df.rating) rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Raw data:") print("Shape: %s" % str(df.shape)) print("Userid size: %d" % n_users) print("Itemid size: %d" % n_items) print("Rating mean: %.5f" % rating_mean) """ Format ratings to user x item matrix """ df = df.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(df, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) print("Raw ratings size", len(ratings)) ratings = ratings.astype(np.float64) """ Construct training data """ # train_size = 0.7 ratings_train_ = ratings #.loc[:int(n_users*train_size), :int(n_items*train_size)] users = ratings_train_.index.values items = ratings_train_.columns.values n_users = len(users) n_items = len(items) temp = ratings_train_.copy() rating_mean = temp.replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Training data:") print("Shape: %s" % str(ratings_train_.shape)) print("n_users: %d" % n_users) print("n_items: %d" % n_items) print("rating mean: %.5f" % rating_mean) user_indices = [x for x in range(n_users)] item_indices = [x for x in range(n_items)] print("Max userid train: ", np.max(users)) print("Max itemid train", np.max(items)) print("user_indices size ", len(user_indices)) print("item_indices size ", len(item_indices)) if normalized_on: ratings_norm = np.zeros(ratings_train_.shape) temp = ratings_train_.values np.subtract(temp, rating_mean, where=temp != 0, out=ratings_norm) ratings = ratings_norm else: ratings = ratings_train_.values # Variables n_features = 10 # latent factors U = tf.Variable(initial_value=tf.truncated_normal([n_users, n_features])) P = tf.Variable(initial_value=tf.truncated_normal([n_features, n_items])) result = tf.matmul(U, P) result_flatten = tf.reshape(result, [-1]) assert (result_flatten.shape[0] == n_users * n_items) R = tf.gather(result_flatten, user_indices[:-1] * n_items + item_indices) assert (R.shape[0] == n_users * n_items) R_ = tf.reshape(R, [tf.div(R.shape[0], n_items), len(item_indices)]) assert (R_.shape == ratings.shape) """ Compute error for values from the original ratings matrix so that means excluding values implicitly computed by UxP """ var = tf.Variable(ratings.astype(np.float32)) compare = tf.not_equal(var, tf.constant(0.0)) compare_op = var.assign(tf.where(compare, tf.ones_like(var), var)) R_masked = tf.multiply(R_, compare_op) assert (ratings.shape == R_masked.shape) """ Cost function: sum_ij{ |r_ij- rhat_ij| + lambda*(|u_i|+|p_j|)} """ diff_op = tf.subtract(ratings.astype(np.float32), R_masked) diff_op_abs = tf.abs(diff_op) base_cost = tf.reduce_sum(diff_op_abs) # Regularizer sum_ij{lambda*(|U_i| + |P_j|)} lambda_ = tf.constant(.001) norm_sums = tf.add(tf.reduce_sum(tf.abs(U)), tf.reduce_sum(tf.abs(P))) regularizer = tf.multiply(norm_sums, lambda_) cost = tf.add(base_cost, regularizer) """ Optimizer """ lr = tf.constant(.0001) global_step = tf.Variable(0, trainable=False) decaying_learning_rate = tf.train.exponential_decay(lr, global_step, 10000, .96, staircase=True) optimizer = tf.train.GradientDescentOptimizer( decaying_learning_rate).minimize(cost, global_step=global_step) """ Run """ init = tf.global_variables_initializer() session.run(init) print("Running stochastic gradient descent..") epoch = 500 for i in range(epoch): session.run(optimizer) if i % 10 == 0 or i == epoch - 1: diff_op_train = tf.subtract(ratings.astype(np.float32), R_masked) diff_op_train_squared = tf.square(diff_op_train) se = tf.reduce_sum(diff_op_train_squared) mse = tf.divide(se, n_users * n_items) rmse = tf.sqrt(mse) print("Train iter: %d MSE: %.5f loss: %.5f" % (i, session.run(rmse), session.run(cost))) R_hat = R_.eval(session=session) predicted_ranks = metrics.rank_matrix(R_hat) interactions = sparse.csr_matrix(ratings) precision = metrics.precision_at_k(predicted_ranks, interactions, k=k) recall = metrics.recall_at_k(predicted_ranks, interactions, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100))
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to n_user x n_item matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Take the mean only from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean:%.3f" % rating_mean) """ Find PQ sub matrices """ R = ratings.values """ Randomly initialize P & Q matrices with n latent factors """ n_factors = 10 P = np.random.normal(0, .1, (n_users, n_factors)) Q = np.random.normal(0, .1, (n_factors, n_items)) R_mask = R.copy() R_mask[R_mask != 0.000000] = 1 R_hat = np.zeros(np.shape(R)) R_hat_mask = np.zeros(np.shape(R)) np.matmul(P, Q, out=R_hat) # Get errors only from explicitly rated elements np.multiply(R_hat, R_mask, out=R_hat_mask) """ Compute error: MSE = (1/N) * (R - R_hat), RMSE = MSE^(1/2) """ diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), n_users * n_items) rmse = np.sqrt(mse) print("RMSE: %.5f" % (rmse)) print("Type: ", type(R_hat)) print(R_hat[:5, :10]) predicted_ranks = metric.rank_matrix(R_hat) print(predicted_ranks.shape) print(predicted_ranks[:5, :10]) ratings_csr = sparse.csr_matrix(ratings.values) precision = metrics.precision_at_k(predicted_ranks, ratings_csr, k=100) recall = metrics.recall_at_k(predicted_ranks, ratings_csr, k=100) print("Precision {0:.5f}% Recall={1:.5f}%".format(precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) normalization_enabled = False optimize_enabled = True k = 100 """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to user x movie matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) """ Construct data """ users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Compute mean ratingonly from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean: %.6f" % rating_mean) R_mask = np.zeros(np.shape(ratings)) R_mask[ratings != 0.000000] = 1 if normalization_enabled: temp = ratings.copy() ratings_norm = np.subtract(temp, rating_mean, where=temp != 0) ratings_norm = np.multiply(ratings_norm, R_mask) assert (np.count_nonzero(ratings_norm) == np.count_nonzero(ratings)) R = ratings_norm.values else: R = ratings.values.copy() # Setup covariance to treat the item columns as input variables covar = np.cov(R, rowvar=False) evals, evecs = np.linalg.eigh(covar) print("cov_mat shape: %s" % str(np.shape(covar))) print("evals shape: %s" % str(np.shape(evals))) print("evecs shape: %s" % str(np.shape(evecs))) n_components = 10 # principal components """ Randomly initialize weights table """ weights = np.random.normal(0, .1, (n_users, n_components)) components = evecs[:n_components, :n_items] R_hat_mask = np.zeros(np.shape(R), dtype=np.float64) if optimize_enabled: # optimization parameters epochs = 5 learning_rate = .0001 lambda_ = .0001 verbosity = 1 print("Optimized PCA epochs=%s" % epochs) """ We only modify the weight matrix """ for epoch in range(epochs): for u in range(n_users): for i in range(n_items): error = R[u, i] - np.dot(weights[u, :], components[:, i]) for k in range(n_components): weights[u, k] = weights[u, k] - learning_rate * ( error * -2 * components[k, i] + lambda_ * (2 * np.abs(weights[u, k]) + 2 * np.abs(components[k, i]))) R_hat = np.zeros(np.shape(R)) np.matmul(weights, components, out=R_hat) # Get errors only from explicitly rated elements np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), np.count_nonzero(R)) rmse = np.sqrt(mse) if epoch % verbosity == 0 or epoch == (epochs - 1): print("Epoch %d: RMSE: %.6f" % (epoch, rmse)) else: R_hat = np.matmul(weights, components) print("R_hat shape: %s" % str(np.shape(R_hat))) assert (np.shape(R) == np.shape(R_hat)) print("PCA single run") np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) mse = np.divide(diff_square.sum(), np.count_nonzero(R)) rmse = np.sqrt(mse) print("RMSE: %.5f" % rmse) assert (R.shape == R_hat.shape) sparse_data = sparse.csr_matrix(R) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, sparse_data, k=k) recall = metrics.recall_at_k(predicted_ranks, sparse_data, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStoppping '%s" % sys.argv[0])
def deepfm_test(self): train_x, train_y = DeepFM.df2xy(self._ratings) #test_x, test_y = DeepFM.df2xy(self.test_data_) params = { 'n_uid': self._ratings.userid.max(), 'n_mid': self._ratings.itemid.max(), # 'n_genre': self.n_genre_, 'k': self._k, 'dnn_dim': [64, 64], 'dnn_dr': 0.5, 'filepath': '../data/deepfm_weights.h5' } """ train """ model = DeepFM(**params) train_history = model.fit(train_x, train_y, epochs=self._epochs, batch_size=2048, validation_split=0.1) history = pd.DataFrame(train_history.history) history.plot() plt.savefig("../data//history.png") """ test """ results = model.evaluate(train_x, train_y) print("Validate result:{0}".format(results)) """ predict """ y_hat = model.predict(train_x) print(np.shape(y_hat)) # print(np.shape(test_y)) """ Run Recall and Precision Metrics """ n_users = np.max(self._ratings.userid.values) + 1 n_items = np.max(self._ratings.itemid.values) + 1 print("n_users={0} n_items={1}".format(n_users, n_items)) # Convert to sparse matrix to run standard metrics sparse_train = sparse.coo_matrix((self._ratings.rating.values, (self._ratings.userid.values, self._ratings.itemid.values)), shape=(n_users, n_items)) # sparse_test = sparse.coo_matrix((self.test_data_.rating.values, \ # (self.test_data_.uid.values, self.test_data_.mid.values)), \ # shape=(n_users, n_items)) # pd.DataFrame(data=sparse_test.tocsr().todense().A).to_csv("./testdata.csv") # test_prediced test_predicted = self._ratings.copy() test_predicted.rating = np.round(y_hat) sparse_predicted = sparse.coo_matrix((test_predicted.rating.values, \ (test_predicted.userid.values, test_predicted.itemid.values)), \ shape=(n_users, n_items)) sparse_train_1up = sparse_train.multiply(sparse_train >= 1) # sparse_test_1up = sparse_test.multiply(sparse_test >= 1) predicted_arr = sparse_predicted.tocsr().todense().A predicted_ranks = metrics.rank_matrix(predicted_arr) precision_ = metrics.precision_at_k(predicted_ranks, sparse_train, k=self._k) recall_ = metrics.recall_at_k(predicted_ranks, sparse_train, k=self._k) print("{0}.xdeepfm_test train precision={1:.4f}% recall={2:.4f}% @k={3}".format( __class__.__name__, precision_ * 100, recall_ * 100, self._k))
def topk_eval(sess, args, data_generator, model, user_list, train_record, eval_record, test_record, item_set, k_list, batch_size, mode='test'): precision_list = {k: [] for k in k_list} recall_list = {k: [] for k in k_list} MAP_list = {k: [] for k in k_list} hit_ratio_list = {k: [] for k in k_list} ndcg_list = {k: [] for k in k_list} for user in user_list: if mode == 'eval': ref_user = eval_record else: ref_user = test_record if user in ref_user: test_item_list = list(item_set - train_record[user]) item_score_map = dict() start = 0 while start + batch_size <= len(test_item_list): user_list_tmp = [user] * batch_size user_list_tmp = np.array(user_list_tmp) item_list = test_item_list[start:start + batch_size] item_list = np.array(item_list) labels_list = [1] * batch_size labels_list = np.array(labels_list) data = np.concatenate((np.expand_dims( user_list_tmp, axis=1), np.expand_dims(item_list, axis=1), np.expand_dims(labels_list, axis=1)), axis=1) batch_data = data_generator.generate_rating_batch( data, 0, args.batch_size) feed_dict = data_generator.generate_feed_rating_dict( model, batch_data) items, scores = model.get_scores(sess, feed_dict) for item, score in zip(items, scores): item_score_map[item] = score start += batch_size # padding the last incomplete minibatch if exists if start < len(test_item_list): user_list_tmp = [user] * batch_size user_list_tmp = np.array(user_list_tmp) item_list = test_item_list[start:] + [test_item_list[-1]] * ( batch_size - len(test_item_list) + start) item_list = np.array(item_list) labels_list = [1] * batch_size labels_list = np.array(labels_list) data = np.concatenate((np.expand_dims( user_list_tmp, axis=1), np.expand_dims(item_list, axis=1), np.expand_dims(labels_list, axis=1)), axis=1) batch_data = data_generator.generate_rating_batch( data, 0, args.batch_size) feed_dict = data_generator.generate_feed_rating_dict( model, batch_data) items, scores = model.get_scores(sess, feed_dict) for item, score in zip(items, scores): item_score_map[item] = score item_score_pair_sorted = sorted(item_score_map.items(), key=lambda x: x[1], reverse=True) item_sorted = [i[0] for i in item_score_pair_sorted] for k in k_list: precision_list[k].append( precision_at_k(item_sorted, ref_user[user], k)) recall_list[k].append( recall_at_k(item_sorted, ref_user[user], k)) # ndcg r_hit = [] for i in item_sorted[:k]: if i in ref_user[user]: r_hit.append(1) else: r_hit.append(0) for k in k_list: ndcg_list[k].append(ndcg_at_k(r_hit, k)) precision = [np.mean(precision_list[k]) for k in k_list] recall = [np.mean(recall_list[k]) for k in k_list] ndcg = [np.mean(ndcg_list[k]) for k in k_list] return precision, recall, ndcg, None, None
# COMMAND ---------- relevance = [] for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID): rec_response = personalize_runtime.get_recommendations( campaignArn = campaign_arn, userId = str(user_id) ) rec_items = [int(x['itemId']) for x in rec_response['itemList']] relevance.append([int(x in true_items.values) for x in rec_items]) # COMMAND ---------- print('mean_reciprocal_rank', np.mean([mean_reciprocal_rank(r) for r in relevance])) print('precision_at_5', np.mean([precision_at_k(r, 5) for r in relevance])) print('precision_at_10', np.mean([precision_at_k(r, 10) for r in relevance])) print('precision_at_25', np.mean([precision_at_k(r, 25) for r in relevance])) print('normalized_discounted_cumulative_gain_at_5', np.mean([ndcg_at_k(r, 5) for r in relevance])) print('normalized_discounted_cumulative_gain_at_10', np.mean([ndcg_at_k(r, 10) for r in relevance])) print('normalized_discounted_cumulative_gain_at_25', np.mean([ndcg_at_k(r, 25) for r in relevance])) # COMMAND ---------- # MAGIC %md # MAGIC ### Optional: slightly better results after deduplicating previous purchase histories # COMMAND ---------- rel_dedup = [] for user_id, true_items in tqdm_notebook(holdout.groupby('USER_ID').ITEM_ID):
def main(): print("\nStarting '%s'" % sys.argv[0]) np.random.seed(8000) k = 100 normalization_enabled = False """ Load dataset """ datafile = "./data/ml-100k/u.data" data = pd.read_csv(datafile, sep='\t', names=["userid", "itemid", "rating", "timestamp"]) """ Convert rating data to user x movie matrix format """ data = data.sort_values(by=["userid", "itemid"]) ratings = pd.pivot_table(data, values="rating", index="userid", columns="itemid") ratings.fillna(0, inplace=True) # train_size = 0.7 # train_row_size = int(len(ratings.index) * train_size) # train_col_size = int(len(ratings.columns) * train_size) # ratings = ratings.loc[:train_row_size, :train_col_size] users = np.unique(ratings.index.values) items = np.unique(ratings.columns.values) n_users = len(users) n_items = len(items) assert (np.max(users) == len(users)) assert (np.max(items) == len(items)) print("n_users=%d n_items=%d" % (n_users, n_items)) """ Take the mean only from non-zero elements """ temp = ratings.copy() rating_mean = temp.copy().replace(0, np.NaN).mean().mean() rating_mean = 3.5 if rating_mean > 3.5 else rating_mean print("Rating mean: %.2f" % rating_mean) if normalization_enabled: temp = ratings.copy() ratings_norm = np.subtract(temp, rating_mean, where=temp != 0) R = ratings_norm.values else: R = ratings.values U, S, V = linalg.svds(R, k=k) # print ("U: ", np.shape(U)) # print ("S: ", np.shape(S)) # print ("V: ", np.shape(V)) sigma = np.diag(S) # print ("Sigma: ", np.shape(sigma)) """ Generate prediction matrix """ R_hat = np.dot(np.dot(U, sigma), V) assert (np.shape(R) == np.shape(R_hat)) # Get errors only from explicitly rated elements R_mask = np.zeros(np.shape(R)) R_mask[R != 0.000000] = 1 R_hat_mask = np.zeros(np.shape(R)) np.multiply(R_hat, R_mask, out=R_hat_mask) # Compute error: MSE = (1/N) * (R - Rˆ), RMSE = MSEˆ(1/2) assert (np.count_nonzero(R) == np.count_nonzero(R_hat_mask)) diff = np.subtract(R, R_hat_mask) diff_square = np.square(diff) #mse = np.divide(diff_square.sum(), n_users*n_items) mse = np.divide(diff_square.sum(), np.count_nonzero(R_mask)) rmse = np.sqrt(mse) print("RMSE: %.6f" % (rmse)) assert (R.shape == R_hat.shape) interactions = sparse.csr_matrix(R) predicted_ranks = metrics.rank_matrix(R_hat) precision = metrics.precision_at_k(predicted_ranks, interactions, k=k) recall = metrics.recall_at_k(predicted_ranks, interactions, k=k) print("Precision:%.3f%% Recall:%.3f%%" % (precision * 100, recall * 100)) print("\nStopping '%s'" % sys.argv[0])