def TestModel(self, conf):
if not os.path.exists(conf['save_path']):
os.makedirs(conf['save_path'])
print('beging test starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(
open(conf["data_path"], 'rb'))
print('finish loading data')
test_batches = reader.build_batches(test_data, conf)
print("finish building test batches")
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
# refine conf
test_batch_num = len(test_batches["response"])
with tf.Session() as sess:
saver = tf.train.Saver()
# with tf.Session() as sess:
# sess.run(init)
saver.restore(sess, os.path.join(conf["init_model"], "model.2"))
print("sucess init %s" % conf["init_model"])
score_file_path = conf['save_path'] + 'score.test'
score_file = open(score_file_path, 'w')
all_candidate_score = []
labels = []
for batch_index in xrange(test_batch_num):
# print('utterance_ph',np.array(test_batches["turns"][batch_index]).shape)
feed_dict = {
self.utterance_ph:
np.array(test_batches["turns"][batch_index]),
# _model.tt_turns_len: test_batches["tt_turns_len"][batch_index],
self.all_utterance_len_ph:
np.array(test_batches["every_turn_len"][batch_index]),
self.response_ph:
np.array(test_batches["response"][batch_index]),
self.response_len:
np.array(test_batches["response_len"][batch_index]),
# _model.label: test_batches["label"][batch_index]
}
# last_hidden = sess.run(self.last_hidden, feed_dict=feed_dict)
# print('last_hidden', last_hidden.shape)
candidate_scores = sess.run(self.y_pred, feed_dict=feed_dict)
all_candidate_score.append(candidate_scores[:, 1])
# scores = sess.run(_model.logits, feed_dict=feed)
for i in xrange(conf["batch_size"]):
score_file.write(
str(candidate_scores[i]) + '\t' +
str(test_batches["label"][batch_index][i]) + '\n')
labels.append(test_batches["label"][batch_index][i])
score_file.close()
all_candidate_scores = np.concatenate(all_candidate_score, axis=0)
Evaluate.ComputeR10_1(all_candidate_scores, labels)
Evaluate.ComputeR2_1(all_candidate_scores, labels)
def TrainModel(self,
conf,
countinue_train=False,
previous_modelpath="model"):
start = time.time()
init = tf.global_variables_initializer()
saver = tf.train.Saver()
merged = tf.summary.merge_all()
print('starting loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
train_data, val_data, test_data = pickle.load(
open(conf["data_path"], 'rb'))
print('finish loading data')
print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
val_batches = reader.build_batches(test_data, conf)
batch_num = len(train_data['y']) / conf[
"batch_size"] # batch_num=12 500 15 625(64的时候)
# val_batch_num = len(val_batches["response"])
print('batch_num', batch_num)
conf["train_steps"] = conf["epoch"] * batch_num # train_steps=2*3906
conf["evaluate_step"] = max(1, batch_num /
1) # max(1,1250) #每隔2500个batch保存一下
conf["print_step"] = max(1, batch_num / 10) # 1250 每隔100个batch打印一下
print('configurations', conf)
with tf.Session() as sess:
writer = tf.summary.FileWriter(conf["output_path"], sess.graph)
train_writer = tf.summary.FileWriter(conf["output_path"],
sess.graph)
with open(conf["embedding_file"], 'rb') as f:
embeddings = pickle.load(f)
if countinue_train == False:
sess.run(init)
sess.run(self.embedding_init,
feed_dict={self.embedding_ph: embeddings})
else:
saver.restore(sess, previous_modelpath)
step = 0
learning_rate = conf['lr']
for step_i in xrange(conf["epoch"]):
print('starting shuffle train data')
shuffle_train = reader.unison_shuffle(train_data) # 打乱
train_batches = reader.build_batches(shuffle_train, conf)
print('finish building train data')
if step_i > 1:
learning_rate = learning_rate
if step_i == 2:
learning_rate = learning_rate * 0.75
if step_i > 2 and (step_i % 2 == 0):
learning_rate = learning_rate * 0.5
for batch_index in range(batch_num):
feed_dict = {
self.utterance_ph:
np.array(train_batches["turns"][batch_index]),
self.all_utterance_len_ph:
np.array(train_batches["every_turn_len"][batch_index]),
self.response_ph:
np.array(train_batches["response"][batch_index]),
self.response_len:
np.array(train_batches["response_len"][batch_index]),
self.y_true:
np.array(train_batches["label"][batch_index]),
self.lr:
learning_rate
}
_, summary = sess.run([self.train_op, merged],
feed_dict=feed_dict)
train_writer.add_summary(summary)
step += 1
if step % conf[
"print_step"] == 0 and step > 0: # print_step=125 一个epoch打印100次
print(
'epoch={i}'.format(i=step_i + 1), 'step:', step,
"loss",
sess.run(self.total_loss, feed_dict=feed_dict),
"processed: [" + str(step * 1.0 / batch_num) + "]")
if step % conf[
"evaluate_step"] == 0 and step > 0: # 12500的倍数就会打印
index = step / conf[
'evaluate_step'] # evaluate_file=1250
score_file_path = conf['save_path'] + 'score.' + str(
index)
self.Evaluate(sess, val_batches, score_file_path)
print('save evaluate_step: %s' % index)
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
if (step_i + 1) > 1: # 模型保存6 8 10
saver.save(
sess,
os.path.join(conf["init_model"],
"model.{0}".format(step_i + 1)))
print(sess.run(self.total_loss, feed_dict=feed_dict))
print('epoch={i} save model'.format(i=step_i))
print('learning rate', learning_rate)
print(
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time())))
end = time.time()
gap = (end - start) / 3600
print('train time:%.4f h' % gap)
