def evaluation(pred_scores, true_scores, samples=10):
'''
:param pred_scores: list of scores predicted by model
:param true_scores: list of ground truth labels, 1 or 0
:return:
'''
num_sample = int(len(pred_scores) / samples) # 1 positive and 9 negative
score_list = np.split(np.array(pred_scores), num_sample, axis=0)
recall_2_1 = recall_2at1(np.array(true_scores), np.array(pred_scores))
recall_at_1 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
1)
recall_at_2 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
2)
recall_at_5 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
5)
_mrr = MRR(np.array(true_scores), np.array(pred_scores))
_map = MAP(np.array(true_scores), np.array(pred_scores))
precision_at_1 = precision_at_k(np.array(true_scores),
np.array(pred_scores),
k=1)
return {
'MAP': _map,
'MRR': _mrr,
'p@1': precision_at_1,
'r2@1': recall_2_1,
'r@1': recall_at_1,
'r@2': recall_at_2,
'r@5': recall_at_5,
}
def dev_step():
acc = []
losses = []
pred_scores = []
ture_scores = []
count = 0
while True:
try:
feed_dict = {
handle: test_handle,
model.dropout_keep_prob: 1.0
}
step, loss, accuracy, y_pred, target = sess.run([
global_step, model.loss, model.accuracy,
model.y_pred, model.target
], feed_dict)
acc.append(accuracy)
losses.append(loss)
pred_scores += list(y_pred[:, 1])
ture_scores += list(target)
count += 1
if count % 1000 == 0:
print("Processing {} samples".format(count))
except tf.errors.OutOfRangeError:
break
MeanAcc = sum(acc) / len(acc)
MeanLoss = sum(losses) / len(losses)
with open(
os.path.join(out_dir,
'predScores-iter-%s-test.txt' % (step)),
'w') as f:
for score1, score2 in zip(pred_scores, ture_scores):
f.writelines(str(score1) + '\t' + str(score2) + '\n')
num_sample = int(len(pred_scores) / 10)
score_list = np.split(np.array(pred_scores),
num_sample,
axis=0)
recall_2_1 = recall_2at1(score_list, k=1)
recall_at_1 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 1)
recall_at_2 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 2)
recall_at_5 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 5)
time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("**********************************")
print('pred_scores: ', len(pred_scores))
print("recall_2_1: %.3f" % (recall_2_1))
print("recall_at_1: %.3f" % (recall_at_1))
print("recall_at_2: %.3f" % (recall_at_2))
print("recall_at_5: %.3f" % (recall_at_5))
print("**********************************")
def dev_step():
acc = []
losses = []
pred_scores = []
ture_scores = []
count = 0
while True:
try:
feed_dict = {
handle: test_handle,
model.is_training: False,
model.dropout_keep_prob: 1.0
}
step, loss, accuracy, y_pred, target = sess.run([
global_step, model.loss, model.accuracy,
model.y_pred, model.target
], feed_dict)
acc.append(accuracy)
losses.append(loss)
pred_scores += list(y_pred[:, 1])
ture_scores += list(target)
count += 1
if count % 1000 == 0:
print(count)
except tf.errors.OutOfRangeError:
break
MeanAcc = sum(acc) / len(acc)
MeanLoss = sum(losses) / len(losses)
if ('ubuntu' in FLAGS.data_path):
num_sample = int(len(pred_scores) / 10)
score_list = np.split(np.array(pred_scores),
num_sample,
axis=0)
recall_2_1 = recall_2at1(score_list, k=1)
recall_at_1 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 1)
recall_at_2 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 2)
recall_at_5 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 5)
time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("**********************************")
print("%s results.........." % (flag.title()))
print('pred_scores: ', len(pred_scores))
print("Step: %d \t| loss: %.3f \t| acc: %.3f \t| %s" %
(step, MeanLoss, MeanAcc, time_str))
print("recall_2_1: %.3f" % (recall_2_1))
print("recall_at_1: %.3f" % (recall_at_1))
print("recall_at_2: %.3f" % (recall_at_2))
print("recall_at_5: %.3f" % (recall_at_5))
print("**********************************")
def evaluation(pred_scores, true_scores, samples=10):
'''
:param pred_scores: list of scores predicted by model
:param true_scores: list of ground truth labels, 1 or 0
:return:
'''
num_sample = int(len(pred_scores) / samples) # 1 positive and 9 negative
# score_list = np.argmax(np.split(np.array(pred_scores), num_sample, axis=0), 1)
# logit_list = np.argmax(np.split(np.array(true_scores), num_sample, axis=0), 1)
recall_2_1 = recall_2at1(np.array(true_scores), np.array(pred_scores))
recall_at_1 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
1)
recall_at_2 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
2)
recall_at_5 = recall_at_k_new(np.array(true_scores), np.array(pred_scores),
5)
_mrr = MRR(np.array(true_scores), np.array(pred_scores))
_map = MAP(np.array(true_scores), np.array(pred_scores))
precision_at_1 = precision_at_k(np.array(true_scores),
np.array(pred_scores),
k=1)
# ndcg_at_1 = NDCG(np.array(true_scores), np.array(pred_scores), 1)
# ndcg_at_2 = NDCG(np.array(true_scores), np.array(pred_scores), 2)
# ndcg_at_5 = NDCG(np.array(true_scores), np.array(pred_scores), 5)
print("**********************************")
print("results..........")
print('pred_scores: ', len(pred_scores))
print("MAP: %.3f" % (_map))
print("MRR: %.3f" % (_mrr))
print("precision_at_1: %.3f" % (precision_at_1))
print("recall_2_1: %.3f" % (recall_2_1))
print("recall_at_1: %.3f" % (recall_at_1))
print("recall_at_2: %.3f" % (recall_at_2))
print("recall_at_5: %.3f" % (recall_at_5))
print("**********************************")
return {
'MAP': _map,
'MRR': _mrr,
'p@1': precision_at_1,
'r2@1': recall_2_1,
'r@1': recall_at_1,
'r@2': recall_at_2,
'r@5': recall_at_5,
}
def dev_step(flag, writer):
sess.run(valid_iterator.initializer)
valid_handle = sess.run(valid_iterator.string_handle())
acc = []
losses = []
pred_scores = []
ture_scores = []
count = 0
while True:
try:
feed_dict = {
handle: valid_handle,
model.is_training: False,
model.dropout_keep_prob: 1.0
}
step, loss, accuracy, y_pred, target = sess.run([
global_step, model.loss, model.accuracy,
model.y_pred, model.target
], feed_dict)
acc.append(accuracy)
losses.append(loss)
pred_scores += list(y_pred[:, 1])
ture_scores += list(target)
count += 1
if count % 1000 == 0:
print(count)
except tf.errors.OutOfRangeError:
break
MeanAcc = sum(acc) / len(acc)
MeanLoss = sum(losses) / len(losses)
with open(
os.path.join(out_dir,
'predScores-iter-%s.txt' % (step)),
'w') as f:
for score1, score2 in zip(pred_scores, ture_scores):
f.writelines(str(score1) + '\t' + str(score2) + '\n')
summary_MeanLoss = tf.Summary(value=[
tf.Summary.Value(tag='%s/MeanLoss' % (flag),
simple_value=MeanLoss)
])
summary_MeanAcc = tf.Summary(value=[
tf.Summary.Value(tag='%s/MeanAcc' % (flag),
simple_value=MeanAcc)
])
writer.add_summary(summary_MeanLoss, step)
writer.add_summary(summary_MeanAcc, step)
if ('ubuntu' in FLAGS.data_path):
num_sample = int(len(pred_scores) / 10)
score_list = np.split(np.array(pred_scores),
num_sample,
axis=0)
recall_2_1 = recall_2at1(score_list, k=1)
recall_at_1 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 1)
recall_at_2 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 2)
recall_at_5 = recall_at_k(np.array(ture_scores),
np.array(pred_scores), 5)
time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print("**********************************")
print("%s results.........." % (flag.title()))
print('pred_scores: ', len(pred_scores))
print("Step: %d \t| loss: %.3f \t| acc: %.3f \t| %s" %
(step, MeanLoss, MeanAcc, time_str))
print("recall_2_1: %.3f" % (recall_2_1))
print("recall_at_1: %.3f" % (recall_at_1))
print("recall_at_2: %.3f" % (recall_at_2))
print("recall_at_5: %.3f" % (recall_at_5))
print("**********************************")
summary_recall_2_1 = tf.Summary(value=[
tf.Summary.Value(tag='%s/recall_2_1' % (flag),
simple_value=recall_2_1)
])
summary_recall_at_1 = tf.Summary(value=[
tf.Summary.Value(tag='%s/recall_at_1' % (flag),
simple_value=recall_at_1)
])
summary_recall_at_2 = tf.Summary(value=[
tf.Summary.Value(tag='%s/recall_at_2' % (flag),
simple_value=recall_at_2)
])
summary_recall_at_5 = tf.Summary(value=[
tf.Summary.Value(tag='%s/recall_at_5' % (flag),
simple_value=recall_at_5)
])
writer.add_summary(summary_recall_2_1, step)
writer.add_summary(summary_recall_at_1, step)
writer.add_summary(summary_recall_at_2, step)
writer.add_summary(summary_recall_at_5, step)
return MeanLoss, recall_2_1 + recall_at_1