def eval(sess, model, u_test_l, u_test_r, save=False):
#ps, rs, ndcgs, mrrs, losses = [],[],[],[],[]
hits, ndcgs, mrrs = [], [], []
number = len(u_test_l)
for u in u_test_l:
test_l, test_r = u_test_l[u], u_test_r[u]
feed_dict = dict()
feed_dict[model.uidx] = test_l
feed_dict[model.iidx] = test_r
predictions = sess.run(model.similarity, feed_dict=feed_dict)
(hr, ndcg, mrr) = eval_one_rating(
predictions[:, 0]
) # get predictions[:,0] because test one user by one user
hits.append(hr)
ndcgs.append(ndcg)
mrrs.append(mrr)
hr = _mean_dict(hits)
ndcg = _mean_dict(ndcgs)
mrr = np.array(mrrs).mean()
print "the number of ui pairs in testset is", number
s = ['HR@' + str(x) for x in hr] + ['NGCD@' + str(x)
for x in ndcg] + ['mrr'] #+ ['loss']
print '\t'.join(s)
s = [str(round(hr[x], 5))
for x in hr] + [str(round(ndcg[x], 5))
for x in ndcg] + [str(round(mrr, 5))
] #+ [str(round(test_loss, 5))]
print "\t".join(s)
def eval(sess, model, data, item2idx, user2idx, items, users, save=False):
#ps, rs, ndcgs, mrrs, losses = [],[],[],[],[]
hits, ndcgs, mrrs, losses = [], [], [], []
#users, items = user2idx.index, item2idx.index
n_copy = model.num_neg + 1
user_negs = data.load_negative()
#UI_simMap = {}
# check the test item with special attrs
#special_attrs = ['film.film.actor', 'film.film.directed_by']
#special_items = data.get_item_with_attrs(special_attrs) # kgid
number = 0
for u in user_negs:
gtItem, cans = user_negs[u]
gtItem = gtItem[0]
cans = [x for x in cans if x in item2idx.index.values]
if gtItem not in items: # or data.item2en[gtItem[0]] in special_items:
continue
number += 1
test_l, test_r = [], []
cansidx = [item2idx[x] for x in cans if x in item2idx]
cansidx.append(item2idx[gtItem])
uidx = user2idx[u]
for iidx in cansidx:
test_l.append([uidx])
test_r.append([iidx] * n_copy)
test_l = np.array(test_l, dtype=np.int32)
test_r = np.array(test_r, dtype=np.int32)
feed_dict = dict()
feed_dict[model.user_input] = test_l
feed_dict[model.item_input] = test_r
#predictions, simMap = sess.run([model.similarity, model.UI_simMap], feed_dict=feed_dict)
predictions = sess.run(model.similarity, feed_dict=feed_dict)
#predictions = sess.run(model.similarity, feed_dict=feed_dict)
#UI_simMap[u] = simMap[:, 0, :]
(hr, ndcg, mrr) = eval_one_rating(
predictions[:, 0]
) # get predictions[:,0] because test one user by one user
hits.append(hr)
ndcgs.append(ndcg)
mrrs.append(mrr)
hr = _mean_dict(hits)
ndcg = _mean_dict(ndcgs)
mrr = np.array(mrrs).mean()
#test_loss = np.array(losses).mean()
print "the number of ui pairs in testset is", number
s = ['HR@' + str(x) for x in hr] + ['NGCD@' + str(x)
for x in ndcg] + ['mrr'] #+ ['loss']
print '\t'.join(s)
s = [str(round(hr[x], 5))
for x in hr] + [str(round(ndcg[x], 5))
for x in ndcg] + [str(round(mrr, 5))
] #+ [str(round(test_loss, 5))]
print "\t".join(s)
def evaluate(sess, model, data, data_generator):
# ------------ test the model -------------- #
hits, ndcgs, mrrs, losses = [], [], [], []
test_user_negs = data.load_negative() # {u:[gtItem, negs]}
for u in test_user_negs:
[gtItem, negs] = test_user_negs[u]
gtItem = gtItem[0]
if gtItem not in data_generator.items:
continue
uidx = data_generator.user2idx[u]
gtidx = data_generator.item2idx[gtItem]
negidxs = [
data_generator.item2idx[x] for x in negs
if x in data_generator.items
]
canidxs = negidxs + [gtidx]
feed_dict = dict()
feed_dict[model.u] = np.array([uidx] * len(canidxs), dtype=np.int32)
feed_dict[model.pos_i] = np.array(canidxs, dtype=np.int32)
predictions = model.evaluate(sess, feed_dict)
(hr, ndcg, mrr) = eval_one_rating(
np.diag(predictions)
) # get predictions[:,0] because test one user by one user
#print hr, ndcg, mrr
#print canidxs
#print predictions
#print feed_dict[model.u].shape, feed_dict[model.pos_i].shape, predictions.shape
hits.append(hr)
ndcgs.append(ndcg)
mrrs.append(mrr)
hr = _mean_dict(hits)
ndcg = _mean_dict(ndcgs)
mrr = np.array(mrrs).mean()
#test_loss = np.array(losses).mean()
s = ['HR@' + str(x) for x in hr] + ['NGCD@' + str(x)
for x in ndcg] + ['mrr'] #+ ['loss']
print '\t'.join(s)
s = [str(round(hr[x], 5))
for x in hr] + [str(round(ndcg[x], 5))
for x in ndcg] + [str(round(mrr, 5))
] #+ [str(round(test_loss, 5))]
print "\t".join(s)