def train():
poem = poemsTxt("./tangshiDataset/dataset/")
trainData, trainSeqLen = poem.load_data()
char2id = poem.char2id
charNums = char2id.__len__()
seqVec = tf.placeholder(tf.int32, [None, None])
seqLen = tf.placeholder(tf.int32, [None])
seqTarget = tf.placeholder(tf.int32, [None, None])
#training
loss = rnn_model(seqVec,
seqTarget,
seqLen,
rnnType="lstm",
hiddenNums=HIDDEN_NUMS,
layerNums=LAYER_NUMS,
batchSize=BATCH_SIZE,
vocaSize=charNums)
#testing
pred, state, init_state = rnn_model(seqVec,
rnnType="lstm",
hiddenNums=HIDDEN_NUMS,
layerNums=LAYER_NUMS,
batchSize=1,
vocaSize=charNums)
Opt = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
nums = int(trainData.shape[0])
for i in range(MAX_ITERATION):
randSample = np.random.randint(0, nums, [BATCH_SIZE])
batch = trainData[randSample, :-1]
batchTarget = trainData[randSample, 1:]
batchLen = trainSeqLen[randSample] - 1
sess.run(Opt,
feed_dict={
seqVec: batch,
seqTarget: batchTarget,
seqLen: batchLen
})
if i % 10 == 0:
LOSS = sess.run(loss,
feed_dict={
seqVec: batch,
seqTarget: batchTarget,
seqLen: batchLen
})
print("Iteration: %d, Loss: %f" % (i, LOSS))
if i % 500 == 0:
vector = generate(sess, seqVec, init_state, state, pred, char2id)
print(poem.vector2sentence(vector))
saver.save(sess, "./save_para/model.ckpt")
def creat_model(self):
self.embed = embed(vocab_size=self.vocab_size,embed_size=self.emb_dim)
self.cnn_x1 = cnn_model(name="cnn_x1",out_size=self.conv_out) #input tensor: [batch, length, embed_size] kernel_size[fiter_size,len_size]
self.cnn_x2 = cnn_model(name="cnn_x2",out_size=self.conv_out)
self.rnn_x1 = rnn_model(name="rnn_x1", cell_size=self.lstm_cell) #input tensor: [batch, length, embed_size]
self.rnn_x2 = rnn_model(name="rnn_x2", cell_size=self.lstm_cell)
self.birnn_x1 = birnn_model(name="birnn_x1")
self.birnn_x2 = birnn_model(name="birnn_x2")
self.Attention_1 = Attention_1(size=32) #input tensor: [batch, length-h,embed_size] ,self attention
self.Attention_2 = Attention_1(name = "Attention_2", size=32)
self.att_mat = Attention_2(name = "att_mat") #input tensor1: [batch, x_1] input tensor2: [batch, x_2], outside attention before conv
self.Att_mat_2 = Attention_2(name="att_mat_2") #outside attention before full connect
self.Att_mat_3 = Attention_2(name = "att_mat_3")
self.fc_layer = FC_model(name= "fc_layer")
def run_training(x_batches, y_batches):
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
# poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path)
# batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int)
# signals = produce_signals()
# x_batches, y_batches = generate_batch(FLAGS.batch_size * FLAGS.time_steps, signals)
# x_batches = tf.reshape(x_batches_,[-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data
# input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None, FLAGS.pulse_size])
# output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None, 1])
input_data = tf.placeholder(tf.float32, [None, FLAGS.time_steps, FLAGS.pulse_size])
output_targets = tf.placeholder(tf.int32, [None, FLAGS.time_steps, 1])
end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=FLAGS.vocab_size,
rnn_size=128, num_layers=2, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
# add by sun
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
if checkpoint:
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
try:
for epoch in range(start_epoch, FLAGS.epochs):
n = 0
n_chunk = len(x_batches) # how many batch
for batch in range(n_chunk):
# process input data
input_batch = np.reshape(x_batches[n], [-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data
output_batch = np.reshape(y_batches[n], [-1, FLAGS.time_steps, 1]) # reshape input_data 这里的第3维度为 1
loss, _, _ = sess.run([
end_points['total_loss'],
end_points['last_state'],
end_points['train_op']
], feed_dict={input_data: input_batch, output_targets: output_batch})
n += 1
print('Epoch: %d, batch: %d, training loss: %.6f' % (epoch, batch, loss))
if epoch % 6 == 0:
saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch)
except KeyboardInterrupt:
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch)
print('## Last epoch were saved, next time will start from epoch {}.'.format(epoch))
def gen_poem(begin_word):
batch_size=1
print("[INFO]loading corpus from %s"%FLAGS.file_path)
peoms_vector,word_int_map,vocabularies=process_poems(FLAGS.file_path)
input_data=tf.placeholder(tf.int32,[batch_size,None])
end_points = rnn_model(model="lstm",input_data=input_data,output_data=None,vocab_size=len(vocabularies),\
rnn_size=128,num_layers=2,batch_size=64,learning_rate=FLAGS.learning_rate)
saver=tf.train.Saver(tf.global_variables())
init_op=tf.group(tf.global_variables_initializer(),tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
checkpoint=tf.train.latest_checkpoint(FLAGS.checkpoints_dir)
saver.restore(sess,checkpoint)
x =np.array([list(map(word_int_map.get,start_token))])
[predict,last_state]=sess.run([end_points['prediction'],end_points['last_state']],\
feed_dict={input_data:x})
if begin_word:
word = begin_word
else:
word=to_word(predict,vocabularies)
poem=""
while word != end_token:
poem +=word
x=np.zeros((1,1))
x[0,0] = word_int_map[word]
[predict,last_state] = sess.run([end_points['prediction'],end_points['last_state']],\
feed_dict={input_data:x,end_points['initial_state']:last_state})
word = to_word(predict,vocabularies)
return poem
def gen_poem(FLAGS, begin_word):
batch_size = 1
print('## loading corpus from %s' % FLAGS.model_dir)
poems_vector, word_int_map, vocabularies = process_poems(FLAGS.file_path)
input_data = tf.placeholder(tf.int32, [batch_size, None])
end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len(
vocabularies), rnn_size=FLAGS.rnn_size, num_layers=FLAGS.num_layers, batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
saver.restore(sess, checkpoint)
x = np.array([list(map(word_int_map.get, start_token))])
# predict相当于output,形状是[batch_size,step,cell_num]
# batch_size = 1,上述形状就变为[step,cell_num]
# predict的形状就如下所示：
# [[0.3,0.9,0.6],
# [0.8,0.5,0.7]]
[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],
feed_dict={input_data: x})
if begin_word:
word = begin_word
else:
word = to_word(predict, vocabularies)
poem_ = ''
i = 0
while word != end_token:
poem_ += word
i += 1
# 生成的诗的字数不超过24
if i >= 24:
break
# x为[[ 0]],x[0, 0]=0
x = np.zeros((1, 1))
# word_int_map[word]的位置index赋值给x[0, 0]，比如‘雨’对应的index为100，那么x[0, 0]初始值就是100
x[0, 0] = word_int_map[word]
[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],
feed_dict={input_data: x, end_points['initial_state']: last_state})
word = to_word(predict, vocabularies)
return poem_
def generate_p(heads=None):
def to_word(predict, words):
sample = np.argmax(predict)
if sample > len(dataset.words):
sample = len(dataset.words) - 1
return dataset.words[sample]
tf.reset_default_graph()
input_data1 = tf.placeholder(tf.int32, [1, None])
end_points = rnn_model(rnn_size=128, nums_layers=2, input_data=input_data1, output_targets=None)
saver = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
ckpt = tf.train.latest_checkpoint(os.getcwd() + '/model/')
saver.restore(sess, ckpt)
peom = ''
x = np.array([list(map(dataset.word_num_map.get, u'['))])
[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x})
word = to_word(predict, dataset.words)
for head in heads:
#x = np.array([list(map(dataset.word_num_map.get, u'['))])
#[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x})
sentence = ''
sentence = head
x[0, 0] = dataset.word_num_map[sentence]
[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x, end_points['initial_state']:last_state})
#x = np.zeros((1, 1))
#word = to_word(predict, dataset.words)
#sentence += word
while word !=u'。':
x = np.zeros((1, 1))
x[0, 0] = dataset.word_num_map[word]
[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x, end_points['initial_state']: last_state})
word = to_word(predict, dataset.words)
sentence += word
peom += sentence
peom += u'\n'
return peom
def run_testing(test_x_batches, test_y_batches):
input_data = tf.placeholder(tf.float32,
[None, FLAGS.time_steps, FLAGS.pulse_size])
# output_targets = tf.placeholder(tf.int32, [None, FLAGS.time_steps, 1])
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=None,
vocab_size=FLAGS.vocab_size,
rnn_size=128,
num_layers=2,
batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(init_op)
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
saver.restore(sess, checkpoint)
n = 0
n_chunk = len(test_x_batches)
for batch in range(n_chunk):
test_input_batch = np.reshape(
test_x_batches[n],
[-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data
test_output_batch = np.reshape(
test_y_batches[n],
[-1, FLAGS.time_steps, 1]) # reshape input_data 这里的第3维度为 1
# print(input_data)
# print(test_input_batch.shape)
[predict, last_state
] = sess.run([end_points['prediction'], end_points['last_state']],
feed_dict={input_data: test_input_batch})
correct_prediction = tf.equal(tf.argmax(predict, 1),
tf.argmax(test_output_batch, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
print("test batch id : ", n, " test accuracy : ",
sess.run(accuracy))
n += 1
def generatePoem(first_char):
seqVec = tf.placeholder(tf.int32, [None, None])
# testing
pred, state, init_state = rnn_model(seqVec,
rnnType="lstm",
hiddenNums=HIDDEN_NUMS,
layerNums=LAYER_NUMS,
batchSize=1,
vocaSize=charNums)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, "./save_para/.\\model.ckpt")
onePoem = generate(sess, seqVec, init_state, state, pred, poem.char2id,
first_char)
print(poem.vector2sentence(onePoem))
def run_training():
if not os.path.exists(os.path.dirname(FLAGS.checkpoints_dir)):
os.mkdir(os.path.dirname(FLAGS.checkpoints_dir))
if not os.path.exists(FLAGS.checkpoints_dir):
os.mkdir(FLAGS.checkpoints_dir)
poems_vector,word_to_int,vocabularies=process_poems(FLAGS.file_path)
batch_inputs,batch_outputs=generate_batch(FLAGS.batch_size,poems_vector,word_to_int)
input_data=tf.placeholder(tf.int32,[FLAGS.batch_size,None])
output_targets=tf.placeholder(tf.int32,[FLAGS.batch_size,None])
end_points=rnn_model(model="lstm",input_data=input_data,\
output_data=output_targets,vocab_size=len(vocabularies),\
rnn_size=128,num_layers=2,batch_size=64,learning_rate=FLAGS.learning_rate)
saver=tf.train.Saver(tf.global_variables())
init_op=tf.group(tf.global_variables_initializer(),tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
start_epoch=0
checkpoint=tf.train.latest_checkpoint(FLAGS.checkpoints_dir)
if checkpoint:
saver.restore(sess,checkpoint)
print("[INFO] restore from the checkpoint{0}".format(checkpoint))
start_epoch +=int(checkpoint.split('-')[-1])
print("[INFO] start trianing...")
try:
for epoch in range(start_epoch,FLAGS.epoches):
n=0
n_chunk = len(poems_vector)//FLAGS.batch_size
for batch in range(n_chunk):
loss,_,_=sess.run([end_points['total_loss'],end_points['last_state'],end_points['train_op']]\
,feed_dict={input_data:batch_inputs[n],output_targets:batch_outputs[n]})
n+=1
print("[INFO]Epoch:%d,batch:%d,training loss:%6f"%(epoch,batch,loss))
if epoch %6==0:
saver.save(sess,os.path.join(FLAGS.checkpoints_dir,FLAGS.model_prefix),global_step=epoch)
except KeyboardInterrupt:
print("[INFO] Interrupt manually,try saving checkpoint for now...")
saver.save(sess,os.path.join(FLAGS.checkpoints_dir,FLAGS.model_prefix),global_step=epoch)
print("[INFO]Last epoch were saved,next time will start from epoch{}".format(epoch))
def train(x_train, x_test, y_train, y_test, args):
'''
Train the RNN network.
'''
# We need to reshape input training data into correct shape:
# number of examples, number of input, dimension of each input.
# It will be done, below in the graph. But, we also need a TF placeholder.
# In our case, number of examples would be the batch_size, which
# is an adjustable parameter, so we assign it to None to signify that.
# Our data is 1D, so 1 was assigned to the 2nd shape dimension.
# The 3rd dimension is the length of protein sequences.
x_input = tf.placeholder(tf.float32, [None, 1, args.n_inputs])
# For target, the size relies again on the batch_size.
target = tf.placeholder(tf.int32, [None])
# Similarly, the test data must be resaped too.
n_test_examples = len(y_test)
x_test = x_test.reshape((n_test_examples, 1, args.n_inputs))
# We call the model and capture the last state in the RNN network.
final_state = rnn_model(x_input, args)
# Calculate probabilities
with tf.name_scope('Logits'):
logits = tf.layers.dense(final_state, args.n_classes)
# Loss function
with tf.name_scope('Loss'):
xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=target, logits=logits)
loss = tf.reduce_mean(xentropy)
# Optimization
with tf.name_scope('Optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate=args.learning_rate)
training_op = optimizer.minimize(loss)
# Accuracy
with tf.name_scope('Accuracy'):
# Whether the targets are in the top K predictions
correct = tf.nn.in_top_k(logits, target, k=1)
accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
# To initialize all TF variables
init = tf.global_variables_initializer()
# Early stopping: if no improvement in test accuracy seen within 10 epochs
# We store the best accuracy and update it if new accuracy after each epoch
# was higher. We'll stop optimization if no improvement seen in 10 epochs.
best_accuracy = 0.0
last_improvement = 0
check_improvement = 10
# Number of examples in the training set
n_train_examples = len(y_train)
# The graph!
with tf.Session() as sess:
init.run()
for epoch in range(args.n_epochs):
for i in range(n_train_examples // args.batch_size):
# Get the trainig batch
X_batch, y_batch = next_batch(x_train, y_train,
args.batch_size)
# Reshape training batch, as explained above, in the shape of the placeholder
X_batch = X_batch.reshape((args.batch_size, 1, args.n_inputs))
sess.run(training_op,
feed_dict={
x_input: X_batch,
target: y_batch
})
# Obtain accuracy on training & test data
acc_train = accuracy.eval(feed_dict={
x_input: X_batch,
target: y_batch
})
acc_test = accuracy.eval(feed_dict={
x_input: x_test,
target: y_test
})
# Update the best accuracy & output accuracy results
if acc_test > best_accuracy:
best_accuracy = acc_test
last_improvement = epoch
print("Epoch", epoch, ":\tTrain accuracy =", acc_train,
";\tTest accuracy =", acc_test)
# Early stopping in case of no improvement.
if epoch - last_improvement > check_improvement:
print("\n#### No improvement seen in ", check_improvement,
" epochs ==> early stopping.\n")
break
def run_training():
if not os.path.exists(os.path.dirname(FLAGS.checkpoints_dir)):
os.mkdir(os.path.dirname(FLAGS.checkpoints_dir))
if not os.path.exists(FLAGS.checkpoints_dir):
os.mkdir(FLAGS.checkpoints_dir)
# 单词转化的数字:向量，单词和数字一一对应的字典，单词
poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path)
# 真实值和目标值
batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size,
poems_vector, word_to_int)
# 数据占位符
input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=64,
learning_rate=FLAGS.learning_rate)
# 实例化保存模型
saver = tf.train.Saver(tf.global_variables())
# 全局变量进行初始化
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
# 先执行，全局变量初始化
sess.run(init_op)
start_epoch = 0
# 把之前训练过的checkpoint拿出来
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoints_dir)
if checkpoint:
# 拿出训练保存模型
saver.restore(sess, checkpoint)
print("[INFO] restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('[INFO] start training...')
try:
for epoch in range(start_epoch, FLAGS.epochs):
n = 0
# 多少行唐诗//每次训练的个数
n_chunk = len(poems_vector) // FLAGS.batch_size
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'], # 损失
end_points['last_state'], # 最后一次输出
end_points['train_op'] # 训练优化损失
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print(
'[INFO] Epoch: %d , batch: %d , training loss: %.6f' %
(epoch, batch, loss))
if epoch % 6 == 0: # 每隔多少次保存
saver.save(sess, FLAGS.checkpoints_dir, global_step=epoch)
except KeyboardInterrupt:
print(
'[INFO] Interrupt manually, try saving checkpoint for now...')
saver.save(sess, FLAGS.checkpoints_dir, global_step=epoch)
print(
'[INFO] Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
def gen_poem(begin_words, num):
batch_size = 1
print('[INFO] loading corpus from %s' % FLAGS.file_path)
# 单词转化的数字:向量，单词和数字一一对应的字典，单词
poems_vector, word_int_map, vocabularies = process_poems(FLAGS.file_path)
# 此时输入为1个
input_data = tf.placeholder(tf.int32, [batch_size, None])
# 损失等
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=None,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=64,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
# 保存模型的位置，拿回sess
checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoints_dir)
# checkpoint = tf.train.latest_checkpoint('./model/')
saver.restore(sess, checkpoint)
# saver.restore(sess,'./model/-24')
# 从字典里面获取到的开始值
x = np.array([list(map(word_int_map.get, start_token))])
[predict, last_state
] = sess.run([end_points['prediction'], end_points['last_state']],
feed_dict={input_data: x})
poem = ''
for begin_word in begin_words:
while True:
if begin_word:
word = begin_word
else:
word = to_word(predict, vocabularies)
sentence = ''
while word != end_token:
sentence += word
x = np.zeros((1, 1))
x[0, 0] = word_int_map[word]
[predict, last_state] = sess.run(
[end_points['prediction'], end_points['last_state']],
feed_dict={
input_data: x,
end_points['initial_state']: last_state
})
word = to_word(predict, vocabularies)
# word = words[np.argmax(probs_)]
if len(sentence) == 2 + 2 * num and ('，' or '？') not in sentence[:num] and ('，' or '？') not in sentence[
num + 1:-1] and \
sentence[num] == '，' and '□' not in sentence:
poem += sentence
# sentence = ''
break
else:
print("我正在写诗呢")
return poem
def run_training(FLAGS):
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
# 产生诗向量，字映射表，字集合
# pems_vector:[[2, 50, 179, 394, 1081, 597, 0, 13, 351, ...], [2, 161, 343, 973, 30, 2455, 0, 155, 297, ...], [2, 10, 89, 13, 644, 1085, 0, 340, 33, ...], [2, 644, 1081, 689, 204, 1116, 0, 25, 1242, ...], [2, 50, 259, 2401, 58, 137, 0, 25, 163, ...], [2, 6, 346, 25, 87, 106, 0, 439, 1926, ...], [2, 2951, 263, 1917, 28, 437, 0, 1155, 219, ...], [2, 238, 173, 873, 847, 552, 0, 1680, 175, ...], [2, 609, 1363, 17, 155, 56, 0, 915, 839, ...], [2, 202, 650, 79, 685, 3330, 0, 780, 539, ...], [2, 13, 4887, 1198, 2139, 699, 0, 1150, 2553, ...], [2, 107, 2126, 1745, 1103, 16, 0, 202, 1399, ...], [2, 89, 119, 296, 768, 173, 0, 200, 187, ...], [2, 539, 764, 630, 17, 386, 0, 435, 816, ...], ...]
# word_to_int:{' ': 6109, '2': 5534, 'B': 2, 'E': 3, 'F': 5355, 'p': 5354, 'ē': 5531, 'ń': 5584, '□': 705, '、': 4110, '。': 1, '】': 5023, '一': 10, '丁': 1484, ...}
# word:('，', '。', 'B', 'E', '不', '人', '山', '风', '日', '无', '一', '云', '花', '春', ...)
poems_vector, word_to_int, words = process_poems(FLAGS.file_path)
# baches_inputs:[array([[ 2, 50, ...ype=int32), array([[ 2, 75, ...ype=int32), array([[ 2, 53, ...ype=int32), array([[ 2, 121, ...ype=int32), array([[ 2, 357, ...ype=int32), array([[ 2, 168, ...ype=int32), array([[ 2, 2483, ...ype=int32), array([[ 2, 2501, ...ype=int32), array([[ 2, 283, ...ype=int32), array([[ 2, 80, ...ype=int32), array([[ 2, 2204, ...ype=int32), array([[ 2, 929, ...ype=int32), array([[ 2, 671, ...ype=int32), array([[ 2, 1440, ...ype=int32), ...]
# batches_outputs:[array([[ 50, 179, ...ype=int32), array([[ 75, 35, ...ype=int32), array([[ 53, 963, ...ype=int32), array([[ 121, 328, ...ype=int32), array([[ 357, 283, ...ype=int32), array([[ 168, 21, ...ype=int32), array([[2483, 641, ...ype=int32), array([[2501, 2176, ...ype=int32), array([[ 283, 11, ...ype=int32), array([[ 80, 846, ...ype=int32), array([[2204, 2024, ...ype=int32), array([[ 929, 2500, ...ype=int32), array([[ 671, 1429, ...ype=int32), array([[1440, 211, ...ype=int32), ...]
batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size,
poems_vector, word_to_int)
# inpuy_data:<tf.Tensor 'Placeholder:0' shape=(64, ?) dtype=int32>
# output_targets:<tf.Tensor 'Placeholder_1:0' shape=(64, ?) dtype=int32>
input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
# 构建模型
# end_points:{'initial_state': (LSTMStateTuple(c=<tf...=float32>), LSTMStateTuple(c=<tf...=float32>)), 'last_state': (LSTMStateTuple(c=<tf...=float32>), LSTMStateTuple(c=<tf...=float32>)), 'loss': <tf.Tensor 'softmax_...e=float32>, 'output': <tf.Tensor 'Reshape:...e=float32>, 'total_loss': <tf.Tensor 'Mean:0' ...e=float32>, 'train_op': <tf.Operation 'Adam'...type=NoOp>}
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(words),
rnn_size=FLAGS.rnn_size,
num_layers=FLAGS.num_layers,
batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate)
# tf里面提供模型保存的是tf.train.Saver()模块。检查checkpoint 内容最好的方法是使用Saver 加载它。
# saver:<tensorflow.python.training.saver.Saver object at 0x7fc7756eaf28>
saver = tf.train.Saver(tf.global_variables())
# 创建一个包含几个操作的op结点,当这个op结点运行完成，所有作为input的ops都被运行完成，这个操作没有返回值
# inti_op:<tf.Operation 'group_deps' type=NoOp>
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
# 写入tensorflow日志
writer = tf.summary.FileWriter(FLAGS.tensorflow_logs, sess.graph)
# 运行以上的节点
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
# 从上次中断的checkpoint开始训练
if checkpoint:
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
file = open(FLAGS.log_path, 'a')
try:
for epoch in range(start_epoch, FLAGS.epochs):
n = 0
# 输入的数据集切成多少块
# n_chunk = len(poems_vector) // FLAGS.batch_size
n_chunk = 50
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'],
end_points['last_state'],
end_points[
'train_op'] # batches_inputs[n]代表取出第几块数据集
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print('Epoch: %d, batch: %d, training loss: %.6f' %
(epoch, batch, loss))
file.write('Epoch: %d, batch: %d, training loss: %.6f' %
(epoch, batch, loss) + "\n")
if epoch % 6 == 0:
saver.save(sess,
os.path.join(FLAGS.model_dir,
FLAGS.model_prefix),
global_step=epoch)
except KeyboardInterrupt:
# 如果Ctrl+c中断，保存checkpoint，
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_prefix),
global_step=epoch)
print(
'## Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
writer.close()
file.close()
file.close()
writer.close()
def train(dataLoader,
validate_after=5,
resume=False,
perform_training=True,
save_best=False,
model_='cnn'):
"""
Perform training and validation of model.
Args:
dataLoader : DataLoader object
validate_after : Number of epochs after which validation is performed.
The model is also saved after this.
resume : If True, a previously saved model file is loaded.
perform_training : If False, training step is skipped, and final testing is done.
save_best : If True, save session for epoch with minimum validation loss.
model_ : String denoting the neural network model to use (RNN or CNN)
"""
model = None
if model_ == 'cnn':
model = models.cnn_model()
elif model_ == 'rnn':
model = models.rnn_model()
sess = tf.Session()
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
if resume:
try:
prev_session = config.resume_ckpt
saver.restore(sess, prev_session)
print("Using previous session: {}".format(prev_session))
except Exception as exp:
print(exp)
print("Creating a new session.")
if save_best:
MIN_VAL_LOSS = 100000000000
if perform_training:
config.init()
train_writer = tf.summary.FileWriter(
os.path.join(config.logdir, "train"), sess.graph)
valid_writer = tf.summary.FileWriter(
os.path.join(config.logdir, "validation"), sess.graph)
for e in range(config.EPOCHS):
epoch_loss = 0.0
for sensor, label in dataLoader.next_train():
# Run the graph.
loss, _, tb = sess.run(
[model['loss'], model['train'], model['summary']],
feed_dict={
model['sensor_data']: sensor,
model['label']: label,
model['training']: True
})
epoch_loss += loss
avg_loss = epoch_loss / dataLoader.train_batches
print("Average loss for epoch {} = {}".format(e, avg_loss))
if e % validate_after == 0:
val_loss = validation(sess, model, dataLoader, valid_writer, e)
if save_best:
if val_loss < MIN_VAL_LOSS:
path = saver.save(sess, config.ckpt, global_step=e)
print("Saved model to {}".format(path))
MIN_VAL_LOSS = val_loss
else:
path = saver.save(sess, config.ckpt, global_step=e)
print("Saved model to {}".format(path))
train_writer.add_summary(tb, e)
print("===========================================")
print("Calculating validation accuracy...")
accuracies = []
positives = negatives = 0
true_positives = true_negatives = false_positives = false_negatives = 0
for sensor, label in dataLoader.next_validation():
# Run the graph.
pred = sess.run(model['prediction'],
feed_dict={
model['sensor_data']: sensor,
model['label']: label,
model['training']: False
})
label = np.argmax(label, axis=1)
positives += np.count_nonzero(label == 1)
negatives += np.count_nonzero(label == 0)
# detects the condition when the condition is present.
true_positives += np.count_nonzero(pred + label == 2)
# does not detect the condition when the condition is absent.
true_negatives += np.count_nonzero(pred + label == 0)
# wrongly indicates that a particular condition or attribute is present.
false_positives += np.count_nonzero(pred > label)
# wrongly indicates that a particular condition or attribute is absent.
false_negatives += np.count_nonzero(pred < label)
accuracies.append(
np.count_nonzero(pred == label) / pred.shape[0] * 100)
accuracies = np.array(accuracies)
# print(positives, negatives)
# print("True positives : {}".format(true_positives))
# print("False negatives: {}".format(false_negatives))
# print("False positives: {}".format(false_positives))
# print("True negatives: {}".format(true_negatives))
print("Sensitivity: {}".format(true_positives / positives))
print("Specificity: {}".format(true_negatives / negatives))
print("Precision: {}".format(true_positives /
(true_positives + false_positives)))
print("Min Validation set accuracy: {} %".format(accuracies.min()))
print("Max Validation set accuracy: {} %".format(accuracies.max()))
print("Average Validation set accuracy: {} %".format(accuracies.mean()))
sess.close()
def run_training():
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path)
batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size,
poems_vector, word_to_int)
input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=64,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
if checkpoint:
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
try:
for epoch in range(start_epoch, FLAGS.epochs):
n = 0
n_chunk = len(poems_vector) // FLAGS.batch_size
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'], end_points['last_state'],
end_points['train_op']
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print('Epoch: %d, batch: %d, training loss: %.6f' %
(epoch, batch, loss))
if epoch % 6 == 0:
saver.save(sess,
os.path.join(FLAGS.model_dir,
FLAGS.model_prefix),
global_step=epoch)
except KeyboardInterrupt:
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_prefix),
global_step=epoch)
print(
'## Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
def run_training():
# 模型保存路径不存在则创建
if not os.path.exists(FLAGS.model_dir):
os.makedirs(FLAGS.model_dir)
# process_poems对古诗进行预处理
poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path)
# batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int)
# 占位向量
input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
# 使用lstm模型进行训练
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=output_targets,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=FLAGS.batch_size,
learning_rate=FLAGS.learning_rate)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
# sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess)
# sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan)
sess.run(init_op)
start_epoch = 0
checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir)
# 如果之前训练过就找回之前的训练结果
if checkpoint:
saver.restore(sess, checkpoint)
print("## restore from the checkpoint {0}".format(checkpoint))
start_epoch += int(checkpoint.split('-')[-1])
print('## start training...')
try:
n_chunk = len(poems_vector) // FLAGS.batch_size
for epoch in range(start_epoch, FLAGS.epochs):
#每次对其中的数据shuffle一次
# print(type(poems_vector))
random.shuffle(poems_vector)
# print(type(poems_vector))
# 这里有每一次输入的训练数据的列表
batches_inputs, batches_outputs = generate_batch(
FLAGS.batch_size, poems_vector, word_to_int)
n = 0
for batch in range(n_chunk):
loss, _, _ = sess.run(
[
end_points['total_loss'], end_points['last_state'],
end_points['train_op']
],
feed_dict={
input_data: batches_inputs[n],
output_targets: batches_outputs[n]
})
n += 1
print('Epoch: %d/%d, batch: %d/%d, training loss: %.6f' %
(epoch, FLAGS.epochs - 1, batch, n_chunk - 1, loss))
if epoch % 6 == 0:
saver.save(sess,
os.path.join(FLAGS.model_dir,
FLAGS.model_prefix),
global_step=epoch)
alltime = time.time() - start_time
print("Time: %d h %d min % ds" %
(alltime // 3600,
(alltime - alltime // 3600 * 3600) // 60, alltime % 60))
except KeyboardInterrupt:
print('## Interrupt manually, try saving checkpoint for now...')
saver.save(sess,
os.path.join(FLAGS.model_dir, FLAGS.model_prefix),
global_step=epoch)
print(
'## Last epoch were saved, next time will start from epoch {}.'
.format(epoch))
def gen_poem(begin_word):
batch_size = 1
print('## loading corpus from %s' % model_dir)
poems_vector, word_int_map, vocabularies = process_poems(corpus_file)
input_data = tf.placeholder(tf.int32, [batch_size, None])
end_points = rnn_model(model='lstm',
input_data=input_data,
output_data=None,
vocab_size=len(vocabularies),
rnn_size=128,
num_layers=2,
batch_size=64,
learning_rate=lr)
saver = tf.train.Saver(tf.global_variables())
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
checkpoint = tf.train.latest_checkpoint(model_dir)
# print(checkpoint)
saver.restore(sess, checkpoint)
x = np.array([list(map(word_int_map.get, start_token))])
[predict, last_state
] = sess.run([end_points['prediction'], end_points['last_state']],
feed_dict={input_data: x})
poem_ = ''
word = begin_word or to_word(predict, vocabularies, poem=poem_)
i = 0
# while not poem_ or ' ' in poem_:
if 1:
print("开始作诗")
poem_ = ''
while word != end_token and word != start_token:
# while 1:
# if word == ' ':
# poem_ = gen_poem(begin_word)
# break
poem_ += word
print(poem_)
i += 1
# if i > 24:
# break
x = np.array([[word_int_map[word]]])
[predict, last_state] = sess.run(
[end_points['prediction'], end_points['last_state']],
feed_dict={
input_data: x,
end_points['initial_state']: last_state
})
word = ' '
# while word == ' ':
if 1:
time.sleep(1)
random.seed(time.time())
word = to_word(predict, vocabularies, poem_)
print('\n')
return poem_
