def Evaluate(self, sess, val_batches, score_file_path):
labels = []
self.all_candidate_scores = []
val_batch_num = len(val_batches["response"])
# eva_score_file = open(score_file_path, 'w')
for batch_index in xrange(val_batch_num):
feed_dict = {
self.utterance_ph:
np.array(val_batches["turns"][batch_index]),
self.all_utterance_len_ph:
np.array(val_batches["every_turn_len"][batch_index]),
self.response_ph:
np.array(val_batches["response"][batch_index]),
self.response_len:
np.array(val_batches["response_len"][batch_index]),
self.y_true:
np.array(val_batches["label"][batch_index])
}
val_loss = sess.run(self.total_loss, feed_dict=feed_dict)
# print('val_loss',val_loss)
candidate_scores = sess.run(self.y_pred, feed_dict=feed_dict)
self.all_candidate_scores.append(candidate_scores[:, 1])
labels.extend(val_batches["label"][batch_index])
# for i in xrange(len(val_batches["label"][batch_index])):
# eva_score_file.write(str(candidate_scores[i]) +'\t'+str(val_batches["label"][batch_index][i])+ '\n')
# eva_score_file.close()
all_candidate_scores = np.concatenate(self.all_candidate_scores,
axis=0)
Evaluate.ComputeR10_1(all_candidate_scores, labels)
Evaluate.ComputeR10_2(all_candidate_scores, labels)
Evaluate.ComputeR10_5(all_candidate_scores, labels)
Evaluate.ComputeR2_1(all_candidate_scores, labels)
def TestModel(self, conf):
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
print('beging test starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(
open(conf["data_path"], 'rb'))
print('finish loading data')
test_batches = reader.build_batches(test_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
# refine conf
test_batch_num = len(test_batches["response"])
with tf.Session() as sess:
saver = tf.train.Saver()
# with tf.Session() as sess:
# sess.run(init)
saver.restore(sess, os.path.join(conf["init_model"], "model.2"))
print("sucess init %s" % conf["init_model"])
score_file_path = conf['save_path'] + 'score.test'
score_file = open(score_file_path, 'w')
all_candidate_score = []
labels = []
for batch_index in xrange(test_batch_num):
# print('utterance_ph',np.array(test_batches["turns"][batch_index]).shape)
feed_dict = {
self.utterance_ph:
np.array(test_batches["turns"][batch_index]),
# _model.tt_turns_len: test_batches["tt_turns_len"][batch_index],
self.all_utterance_len_ph:
np.array(test_batches["every_turn_len"][batch_index]),
self.response_ph:
np.array(test_batches["response"][batch_index]),
self.response_len:
np.array(test_batches["response_len"][batch_index]),
# _model.label: test_batches["label"][batch_index]
}
# last_hidden = sess.run(self.last_hidden, feed_dict=feed_dict)
# print('last_hidden', last_hidden.shape)
candidate_scores = sess.run(self.y_pred, feed_dict=feed_dict)
all_candidate_score.append(candidate_scores[:, 1])
# scores = sess.run(_model.logits, feed_dict=feed)
for i in xrange(conf["batch_size"]):
score_file.write(
str(candidate_scores[i]) + '\t' +
str(test_batches["label"][batch_index][i]) + '\n')
labels.append(test_batches["label"][batch_index][i])
score_file.close()
all_candidate_scores = np.concatenate(all_candidate_score, axis=0)
Evaluate.ComputeR10_1(all_candidate_scores, labels)
Evaluate.ComputeR2_1(all_candidate_scores, labels)
def Evaluate(self, sess):
with open(ev_path, 'rb') as f:
history, true_utt, labels = pickle.load(f)
with open(ev_char_path, 'rb') as f:
utt_char, true_ch_utt, ch_labels = pickle.load(f)
self.all_candidate_scores = []
history, history_len = utils.multi_sequences_padding(
history, self.max_sentence_len)
history, history_len = np.array(history), np.array(history_len)
true_utt_len = np.array(
utils.get_sequences_length(true_utt, maxlen=self.max_sentence_len))
true_utt = np.array(
pad_sequences(true_utt,
padding='post',
maxlen=self.max_sentence_len))
utt_char = multi_char_sequences_padding(utt_char, 50)
true_ch_utt = np.array(
pad_sequences(true_ch_utt,
padding='post',
maxlen=self.max_sentence_len))
low = 0
dro = 0.1
while True:
feed_dict = {
self.utterance_ph:
np.concatenate([history[low:low + 200]], axis=0),
self.all_utterance_len_ph:
np.concatenate([history_len[low:low + 200]], axis=0),
self.response_ph:
np.concatenate([true_utt[low:low + 200]], axis=0),
self.response_len:
np.concatenate([true_utt_len[low:low + 200]], axis=0),
self.response_cph:
np.concatenate([true_ch_utt[low:low + 200]],
axis=0), # todo negs
self.utterance_cph:
np.concatenate([utt_char[low:low + 200]], axis=0),
self.dropout:
dro,
self.N:
200,
self.sample_numbers:
1
}
candidate_scores = sess.run(self.y_pred, feed_dict=feed_dict)
self.all_candidate_scores.append(candidate_scores[:, 1])
low = low + 200
if low >= history.shape[0]:
break
all_candidate_scores = np.concatenate(self.all_candidate_scores,
axis=0)
computeR10_1 = Evaluate.ComputeR10_1(all_candidate_scores, labels)
computeR2_1 = Evaluate.ComputeR2_1(all_candidate_scores, labels)
return computeR10_1, computeR2_1
def valid():
all_scores = []
for mini_batch in validloader:
query, response = mini_batch[0], mini_batch[1]
query = query.long().to(device)
response = response.long().to(device)
logit = model(query, response).squeeze()
scores = F.softmax(logit, 0).cpu().detach().numpy()
all_scores.append(np.argmax(scores[:, 1]))
#all_scores = np.concatenate(all_scores,axis=0)
print(np.size(all_scores), np.size(labels))
return Evaluate.ComputeR10_1(all_scores, labels)
def Evaluate(self,sess):
with open(evaluate_file, 'rb') as f:
history, true_utt,labels = pickle.load(f)
self.all_candidate_scores = []
history, history_len = utils.multi_sequences_padding(history, self.max_sentence_len)
history, history_len = np.array(history), np.array(history_len)
true_utt_len = np.array(utils.get_sequences_length(true_utt, maxlen=self.max_sentence_len))
true_utt = np.array(pad_sequences(true_utt, padding='post', maxlen=self.max_sentence_len))
low = 0
while True:
feed_dict = {self.utterance_ph: np.concatenate([history[low:low + 200]], axis=0),
self.all_utterance_len_ph: np.concatenate([history_len[low:low + 200]], axis=0),
self.response_ph: np.concatenate([true_utt[low:low + 200]], axis=0),
self.response_len: np.concatenate([true_utt_len[low:low + 200]], axis=0),
}
candidate_scores = sess.run(self.y_pred, feed_dict=feed_dict)
self.all_candidate_scores.append(candidate_scores[:, 1])
low = low + 200
if low >= history.shape[0]:
break
all_candidate_scores = np.concatenate(self.all_candidate_scores, axis=0)
Evaluate.ComputeR10_1(all_candidate_scores,labels)
Evaluate.ComputeR2_1(all_candidate_scores,labels)
