def train():
print("重新训练，请保证计算机拥有至少8G空闲内存与2G空闲显存")
# 准备训练数据
print("正在准备训练数据，大约需要五分钟...")
qTrain, aTrain, lTrain, qIdTrain, aIdTrain = qaData.loadjsonData(
trainingFile, word2idx, unrollSteps, True)
print("训练数据准备完毕")
tqs, tta, tfa = [], [], []
for question, trueAnswer, falseAnswer in qaData.trainingBatchIter(
qTrain, aTrain, lTrain, qIdTrain, batchSize):
tqs.append(question), tta.append(trueAnswer), tfa.append(falseAnswer)
print("训练数据加载完成！")
# 开始训练
print("开始训练，全部训练过程大约需要12小时")
sess.run(tf.global_variables_initializer())
lr = learningRate # 引入局部变量，防止shadow name
for i in range(lrDownCount):
optimizer = tf.train.GradientDescentOptimizer(lr)
optimizer.apply_gradients(zip(grads, tvars))
trainOp = optimizer.apply_gradients(zip(grads, tvars),
global_step=globalStep)
for epoch in range(epochs):
print("epoch", epoch)
for question, trueAnswer, falseAnswer in zip(tqs, tta, tfa):
# print("question.shape = ", question.shape)
# print("trueAnswer.shape = ", trueAnswer.shape)
# print("falseAnswer.shape = ", falseAnswer.shape)
startTime = time.time()
feed_dict = {
lstm.inputQuestions: question,
lstm.inputTrueAnswers: trueAnswer,
lstm.inputFalseAnswers: falseAnswer,
lstm.keep_prob: dropout
}
summary_val = sess.run(lstm.dev_summary_op, feed_dict)
sess.run(trainOp, feed_dict)
step = sess.run(globalStep, feed_dict)
sess.run(lstm.trueCosSim, feed_dict)
sess.run(lstm.falseCosSim, feed_dict)
loss = sess.run(lstm.loss, feed_dict)
timeUsed = time.time() - startTime
print("step:", step, "loss:", loss, "time:", timeUsed)
saver.save(sess, saveFile)
lr *= lrDownRate
def train():
print("重新训练，请保证计算机拥有至少8G空闲内存与2G空闲显存")
# 准备训练数据
print("正在准备训练数据，大约需要五分钟...")
qTrain, aTrain, lTrain, qIdTrain, aIdTrain = qaData.loadjsonData(
trainingFile, word2idx, unrollSteps, True)
#qDevelop, aDevelop, lDevelop, qIdDevelop = qaData.loadData(developFile, word2idx, unrollSteps, True)
#trainQuestionCounts = len(qIdTrain)-1
#for i in range(len(qIdDevelop)):
# qIdDevelop[i] += trainQuestionCounts
tqs, tta, tfa = [], [], []
for question, trueAnswer, falseAnswer in qaData.trainingBatchIter(
qTrain, aTrain, lTrain, qIdTrain, batchSize):
tqs.append(question), tta.append(trueAnswer), tfa.append(falseAnswer)
print("加载完成！")
# 开始训练
print("开始训练，全部训练过程大约需要12小时")
sess.run(tf.global_variables_initializer())
lr = learningRate # 引入局部变量，防止shadow name
for i in range(lrDownCount):
optimizer = tf.train.GradientDescentOptimizer(lr)
optimizer.apply_gradients(zip(grads, tvars))
trainOp = optimizer.apply_gradients(zip(grads, tvars),
global_step=globalStep)
for epoch in range(epochs):
print("epoch", epoch)
for question, trueAnswer, falseAnswer in zip(tqs, tta, tfa):
startTime = time.time()
feed_dict = {
lstm.inputQuestions: question,
lstm.inputTrueAnswers: trueAnswer,
lstm.inputFalseAnswers: falseAnswer,
lstm.keep_prob: dropout
}
summary_val,_, step, _, _, loss = \
sess.run([lstm.dev_summary_op,trainOp, globalStep, lstm.trueCosSim, lstm.falseCosSim, lstm.loss], feed_dict)
timeUsed = time.time() - startTime
if step % 120 == 119:
train_writer = tf.summary.FileWriter(
'/Users/cjf/Downloads/log', sess.graph)
train_writer.add_summary(summary_val, step)
train_writer.close()
print("step:", step, "loss:", loss, "time:", timeUsed)
saver.save(sess, saveFile)
lr *= lrDownRate
def train():
print("重新训练，请保证计算机拥有至少8G空闲内存与2G空闲显存")
# 准备训练数据
print("正在准备训练数据，大约需要五分钟...")
qTrain, aTrain, lTrain, qIdTrain = qaData.loadData(trainingFile, word2idx, unrollSteps, True)
qDevelop, aDevelop, lDevelop, qIdDevelop = qaData.loadData(developFile, word2idx, unrollSteps, True)
trainQuestionCounts = qIdTrain[-1]
for i in range(len(qIdDevelop)):
qIdDevelop[i] += trainQuestionCounts
tqs, tta, tfa = [], [], []
for question, trueAnswer, falseAnswer in qaData.trainingBatchIter(qTrain + qDevelop, aTrain + aDevelop,
lTrain + lDevelop, qIdTrain + qIdDevelop,
batchSize):
tqs.append(question), tta.append(trueAnswer), tfa.append(falseAnswer)
print("加载完成！")
# 开始训练
print("开始训练，全部训练过程大约需要12小时")
sess.run(tf.global_variables_initializer())
lr = learningRate # 引入局部变量，防止shadow name
for i in range(lrDownCount):
optimizer = tf.train.GradientDescentOptimizer(lr)
optimizer.apply_gradients(zip(grads, tvars))
trainOp = optimizer.apply_gradients(zip(grads, tvars), global_step=globalStep)
for epoch in range(epochs):
for question, trueAnswer, falseAnswer in zip(tqs, tta, tfa):
startTime = time.time()
feed_dict = {
lstm.inputQuestions: question,
lstm.inputTrueAnswers: trueAnswer,
lstm.inputFalseAnswers: falseAnswer,
lstm.keep_prob: dropout
}
_, step, _, _, loss = \
sess.run([trainOp, globalStep, lstm.trueCosSim, lstm.falseCosSim, lstm.loss], feed_dict)
timeUsed = time.time() - startTime
print("step:", step, "loss:", loss, "time:", timeUsed)
saver.save(sess, saveFile)
lr *= lrDownRate
def train():
print("重新训练，请保证计算机拥有至少8G空闲内存与2G空闲显存")
# 准备训练数据
print("正在准备训练数据，大约需要五分钟...")
qTrain, aTrain, lTrain, qIdTrain = qaData.loadData(trainingFile, word2idx,
unrollSteps, True)
tqs, tta, tfa = [], [], []
for question, trueAnswer, falseAnswer in qaData.trainingBatchIter(
qTrain, aTrain, lTrain, qIdTrain, batchSize):
tqs.append(question), tta.append(trueAnswer), tfa.append(falseAnswer)
tqs = np.array(tqs)
tta = np.array(tta)
tfa = np.array(tfa)
print("加载完成！")
np.random.seed(10)
shuffle_indices = np.random.permutation(np.arange(len(tqs)))
tqs_shuffled = tqs[shuffle_indices]
tta_shuffled = tta[shuffle_indices]
tfa_shuffled = tfa[shuffle_indices]
# Split train/test set
# TODO: This is very crude, should use cross-validation
dev_sample_index = -1 * int(0.1 * float(len(tqs)))
tqs_train, tqs_dev = tqs_shuffled[:dev_sample_index], tqs_shuffled[
dev_sample_index:]
tta_train, tta_dev = tta_shuffled[:dev_sample_index], tta_shuffled[
dev_sample_index:]
tfa_train, tfa_dev = tfa_shuffled[:dev_sample_index], tfa_shuffled[
dev_sample_index:]
print("开始训练，全部训练过程大约需要12小时")
sess.run(tf.global_variables_initializer())
lr = learningRate # 引入局部变量，防止shadow name
for i in range(lrDownCount):
optimizer = tf.train.GradientDescentOptimizer(lr)
optimizer.apply_gradients(zip(grads, tvars))
trainOp = optimizer.apply_gradients(zip(grads, tvars),
global_step=globalStep)
for epoch in range(epochs):
for question, trueAnswer, falseAnswer in zip(
tqs_train, tta_train, tfa_train):
startTime = time.time()
feed_dict = {
lstm.inputQuestions: question,
lstm.inputTrueAnswers: trueAnswer,
lstm.inputFalseAnswers: falseAnswer,
lstm.keep_prob: dropout
}
_, step, _, _, loss = \
sess.run([trainOp, globalStep, lstm.trueCosSim, lstm.falseCosSim, lstm.loss], feed_dict)
timeUsed = time.time() - startTime
print("step:", step, "loss:", loss, "time:", timeUsed)
saver.save(sess, saveFile)
lr *= lrDownRate
with open(resultFile, 'w') as file:
for question, answer in zip(tqs_dev, tta_dev):
feed_dict = {
lstm.inputTestQuestions: question,
lstm.inputTestAnswers: answer,
lstm.keep_prob: dropout
}
_, scores = sess.run([globalStep, lstm.result], feed_dict)
for score in scores:
print("%.9f" % score + '\n')
print("------------------------------------------")
for question, answer in zip(tqs_dev, tfa_dev):
feed_dict = {
lstm.inputTestQuestions: question,
lstm.inputTestAnswers: answer,
lstm.keep_prob: dropout
}
_, scores = sess.run([globalStep, lstm.result], feed_dict)
for score in scores:
print("%.9f" % score + '\n')