def train(): poem = poemsTxt("./tangshiDataset/dataset/") trainData, trainSeqLen = poem.load_data() char2id = poem.char2id charNums = char2id.__len__() seqVec = tf.placeholder(tf.int32, [None, None]) seqLen = tf.placeholder(tf.int32, [None]) seqTarget = tf.placeholder(tf.int32, [None, None]) #training loss = rnn_model(seqVec, seqTarget, seqLen, rnnType="lstm", hiddenNums=HIDDEN_NUMS, layerNums=LAYER_NUMS, batchSize=BATCH_SIZE, vocaSize=charNums) #testing pred, state, init_state = rnn_model(seqVec, rnnType="lstm", hiddenNums=HIDDEN_NUMS, layerNums=LAYER_NUMS, batchSize=1, vocaSize=charNums) Opt = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() nums = int(trainData.shape[0]) for i in range(MAX_ITERATION): randSample = np.random.randint(0, nums, [BATCH_SIZE]) batch = trainData[randSample, :-1] batchTarget = trainData[randSample, 1:] batchLen = trainSeqLen[randSample] - 1 sess.run(Opt, feed_dict={ seqVec: batch, seqTarget: batchTarget, seqLen: batchLen }) if i % 10 == 0: LOSS = sess.run(loss, feed_dict={ seqVec: batch, seqTarget: batchTarget, seqLen: batchLen }) print("Iteration: %d, Loss: %f" % (i, LOSS)) if i % 500 == 0: vector = generate(sess, seqVec, init_state, state, pred, char2id) print(poem.vector2sentence(vector)) saver.save(sess, "./save_para/model.ckpt")
def creat_model(self): self.embed = embed(vocab_size=self.vocab_size,embed_size=self.emb_dim) self.cnn_x1 = cnn_model(name="cnn_x1",out_size=self.conv_out) #input tensor: [batch, length, embed_size] kernel_size[fiter_size,len_size] self.cnn_x2 = cnn_model(name="cnn_x2",out_size=self.conv_out) self.rnn_x1 = rnn_model(name="rnn_x1", cell_size=self.lstm_cell) #input tensor: [batch, length, embed_size] self.rnn_x2 = rnn_model(name="rnn_x2", cell_size=self.lstm_cell) self.birnn_x1 = birnn_model(name="birnn_x1") self.birnn_x2 = birnn_model(name="birnn_x2") self.Attention_1 = Attention_1(size=32) #input tensor: [batch, length-h,embed_size] ,self attention self.Attention_2 = Attention_1(name = "Attention_2", size=32) self.att_mat = Attention_2(name = "att_mat") #input tensor1: [batch, x_1] input tensor2: [batch, x_2], outside attention before conv self.Att_mat_2 = Attention_2(name="att_mat_2") #outside attention before full connect self.Att_mat_3 = Attention_2(name = "att_mat_3") self.fc_layer = FC_model(name= "fc_layer")
def run_training(x_batches, y_batches): if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) # poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path) # batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int) # signals = produce_signals() # x_batches, y_batches = generate_batch(FLAGS.batch_size * FLAGS.time_steps, signals) # x_batches = tf.reshape(x_batches_,[-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data # input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None, FLAGS.pulse_size]) # output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None, 1]) input_data = tf.placeholder(tf.float32, [None, FLAGS.time_steps, FLAGS.pulse_size]) output_targets = tf.placeholder(tf.int32, [None, FLAGS.time_steps, 1]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=FLAGS.vocab_size, rnn_size=128, num_layers=2, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) # add by sun config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess) # sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) sess.run(init_op) start_epoch = 0 checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) if checkpoint: saver.restore(sess, checkpoint) print("## restore from the checkpoint {0}".format(checkpoint)) start_epoch += int(checkpoint.split('-')[-1]) print('## start training...') try: for epoch in range(start_epoch, FLAGS.epochs): n = 0 n_chunk = len(x_batches) # how many batch for batch in range(n_chunk): # process input data input_batch = np.reshape(x_batches[n], [-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data output_batch = np.reshape(y_batches[n], [-1, FLAGS.time_steps, 1]) # reshape input_data 这里的第3维度为 1 loss, _, _ = sess.run([ end_points['total_loss'], end_points['last_state'], end_points['train_op'] ], feed_dict={input_data: input_batch, output_targets: output_batch}) n += 1 print('Epoch: %d, batch: %d, training loss: %.6f' % (epoch, batch, loss)) if epoch % 6 == 0: saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) except KeyboardInterrupt: print('## Interrupt manually, try saving checkpoint for now...') saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) print('## Last epoch were saved, next time will start from epoch {}.'.format(epoch))
def gen_poem(begin_word): batch_size=1 print("[INFO]loading corpus from %s"%FLAGS.file_path) peoms_vector,word_int_map,vocabularies=process_poems(FLAGS.file_path) input_data=tf.placeholder(tf.int32,[batch_size,None]) end_points = rnn_model(model="lstm",input_data=input_data,output_data=None,vocab_size=len(vocabularies),\ rnn_size=128,num_layers=2,batch_size=64,learning_rate=FLAGS.learning_rate) saver=tf.train.Saver(tf.global_variables()) init_op=tf.group(tf.global_variables_initializer(),tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) checkpoint=tf.train.latest_checkpoint(FLAGS.checkpoints_dir) saver.restore(sess,checkpoint) x =np.array([list(map(word_int_map.get,start_token))]) [predict,last_state]=sess.run([end_points['prediction'],end_points['last_state']],\ feed_dict={input_data:x}) if begin_word: word = begin_word else: word=to_word(predict,vocabularies) poem="" while word != end_token: poem +=word x=np.zeros((1,1)) x[0,0] = word_int_map[word] [predict,last_state] = sess.run([end_points['prediction'],end_points['last_state']],\ feed_dict={input_data:x,end_points['initial_state']:last_state}) word = to_word(predict,vocabularies) return poem
def gen_poem(FLAGS, begin_word): batch_size = 1 print('## loading corpus from %s' % FLAGS.model_dir) poems_vector, word_int_map, vocabularies = process_poems(FLAGS.file_path) input_data = tf.placeholder(tf.int32, [batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len( vocabularies), rnn_size=FLAGS.rnn_size, num_layers=FLAGS.num_layers, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) saver.restore(sess, checkpoint) x = np.array([list(map(word_int_map.get, start_token))]) # predict相当于output,形状是[batch_size,step,cell_num] # batch_size = 1,上述形状就变为[step,cell_num] # predict的形状就如下所示: # [[0.3,0.9,0.6], # [0.8,0.5,0.7]] [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x}) if begin_word: word = begin_word else: word = to_word(predict, vocabularies) poem_ = '' i = 0 while word != end_token: poem_ += word i += 1 # 生成的诗的字数不超过24 if i >= 24: break # x为[[ 0]],x[0, 0]=0 x = np.zeros((1, 1)) # word_int_map[word]的位置index赋值给x[0, 0],比如‘雨’对应的index为100,那么x[0, 0]初始值就是100 x[0, 0] = word_int_map[word] [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x, end_points['initial_state']: last_state}) word = to_word(predict, vocabularies) return poem_
def generate_p(heads=None): def to_word(predict, words): sample = np.argmax(predict) if sample > len(dataset.words): sample = len(dataset.words) - 1 return dataset.words[sample] tf.reset_default_graph() input_data1 = tf.placeholder(tf.int32, [1, None]) end_points = rnn_model(rnn_size=128, nums_layers=2, input_data=input_data1, output_targets=None) saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() ckpt = tf.train.latest_checkpoint(os.getcwd() + '/model/') saver.restore(sess, ckpt) peom = '' x = np.array([list(map(dataset.word_num_map.get, u'['))]) [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x}) word = to_word(predict, dataset.words) for head in heads: #x = np.array([list(map(dataset.word_num_map.get, u'['))]) #[predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x}) sentence = '' sentence = head x[0, 0] = dataset.word_num_map[sentence] [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x, end_points['initial_state']:last_state}) #x = np.zeros((1, 1)) #word = to_word(predict, dataset.words) #sentence += word while word !=u'。': x = np.zeros((1, 1)) x[0, 0] = dataset.word_num_map[word] [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']],feed_dict={input_data1: x, end_points['initial_state']: last_state}) word = to_word(predict, dataset.words) sentence += word peom += sentence peom += u'\n' return peom
def run_testing(test_x_batches, test_y_batches): input_data = tf.placeholder(tf.float32, [None, FLAGS.time_steps, FLAGS.pulse_size]) # output_targets = tf.placeholder(tf.int32, [None, FLAGS.time_steps, 1]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=FLAGS.vocab_size, rnn_size=128, num_layers=2, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: sess.run(init_op) checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) saver.restore(sess, checkpoint) n = 0 n_chunk = len(test_x_batches) for batch in range(n_chunk): test_input_batch = np.reshape( test_x_batches[n], [-1, FLAGS.time_steps, FLAGS.pulse_size]) # reshape input_data test_output_batch = np.reshape( test_y_batches[n], [-1, FLAGS.time_steps, 1]) # reshape input_data 这里的第3维度为 1 # print(input_data) # print(test_input_batch.shape) [predict, last_state ] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: test_input_batch}) correct_prediction = tf.equal(tf.argmax(predict, 1), tf.argmax(test_output_batch, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) print("test batch id : ", n, " test accuracy : ", sess.run(accuracy)) n += 1
def generatePoem(first_char): seqVec = tf.placeholder(tf.int32, [None, None]) # testing pred, state, init_state = rnn_model(seqVec, rnnType="lstm", hiddenNums=HIDDEN_NUMS, layerNums=LAYER_NUMS, batchSize=1, vocaSize=charNums) sess = tf.Session() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess, "./save_para/.\\model.ckpt") onePoem = generate(sess, seqVec, init_state, state, pred, poem.char2id, first_char) print(poem.vector2sentence(onePoem))
def run_training(): if not os.path.exists(os.path.dirname(FLAGS.checkpoints_dir)): os.mkdir(os.path.dirname(FLAGS.checkpoints_dir)) if not os.path.exists(FLAGS.checkpoints_dir): os.mkdir(FLAGS.checkpoints_dir) poems_vector,word_to_int,vocabularies=process_poems(FLAGS.file_path) batch_inputs,batch_outputs=generate_batch(FLAGS.batch_size,poems_vector,word_to_int) input_data=tf.placeholder(tf.int32,[FLAGS.batch_size,None]) output_targets=tf.placeholder(tf.int32,[FLAGS.batch_size,None]) end_points=rnn_model(model="lstm",input_data=input_data,\ output_data=output_targets,vocab_size=len(vocabularies),\ rnn_size=128,num_layers=2,batch_size=64,learning_rate=FLAGS.learning_rate) saver=tf.train.Saver(tf.global_variables()) init_op=tf.group(tf.global_variables_initializer(),tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) start_epoch=0 checkpoint=tf.train.latest_checkpoint(FLAGS.checkpoints_dir) if checkpoint: saver.restore(sess,checkpoint) print("[INFO] restore from the checkpoint{0}".format(checkpoint)) start_epoch +=int(checkpoint.split('-')[-1]) print("[INFO] start trianing...") try: for epoch in range(start_epoch,FLAGS.epoches): n=0 n_chunk = len(poems_vector)//FLAGS.batch_size for batch in range(n_chunk): loss,_,_=sess.run([end_points['total_loss'],end_points['last_state'],end_points['train_op']]\ ,feed_dict={input_data:batch_inputs[n],output_targets:batch_outputs[n]}) n+=1 print("[INFO]Epoch:%d,batch:%d,training loss:%6f"%(epoch,batch,loss)) if epoch %6==0: saver.save(sess,os.path.join(FLAGS.checkpoints_dir,FLAGS.model_prefix),global_step=epoch) except KeyboardInterrupt: print("[INFO] Interrupt manually,try saving checkpoint for now...") saver.save(sess,os.path.join(FLAGS.checkpoints_dir,FLAGS.model_prefix),global_step=epoch) print("[INFO]Last epoch were saved,next time will start from epoch{}".format(epoch))
def train(x_train, x_test, y_train, y_test, args): ''' Train the RNN network. ''' # We need to reshape input training data into correct shape: # number of examples, number of input, dimension of each input. # It will be done, below in the graph. But, we also need a TF placeholder. # In our case, number of examples would be the batch_size, which # is an adjustable parameter, so we assign it to None to signify that. # Our data is 1D, so 1 was assigned to the 2nd shape dimension. # The 3rd dimension is the length of protein sequences. x_input = tf.placeholder(tf.float32, [None, 1, args.n_inputs]) # For target, the size relies again on the batch_size. target = tf.placeholder(tf.int32, [None]) # Similarly, the test data must be resaped too. n_test_examples = len(y_test) x_test = x_test.reshape((n_test_examples, 1, args.n_inputs)) # We call the model and capture the last state in the RNN network. final_state = rnn_model(x_input, args) # Calculate probabilities with tf.name_scope('Logits'): logits = tf.layers.dense(final_state, args.n_classes) # Loss function with tf.name_scope('Loss'): xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=target, logits=logits) loss = tf.reduce_mean(xentropy) # Optimization with tf.name_scope('Optimizer'): optimizer = tf.train.AdamOptimizer(learning_rate=args.learning_rate) training_op = optimizer.minimize(loss) # Accuracy with tf.name_scope('Accuracy'): # Whether the targets are in the top K predictions correct = tf.nn.in_top_k(logits, target, k=1) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32)) # To initialize all TF variables init = tf.global_variables_initializer() # Early stopping: if no improvement in test accuracy seen within 10 epochs # We store the best accuracy and update it if new accuracy after each epoch # was higher. We'll stop optimization if no improvement seen in 10 epochs. best_accuracy = 0.0 last_improvement = 0 check_improvement = 10 # Number of examples in the training set n_train_examples = len(y_train) # The graph! with tf.Session() as sess: init.run() for epoch in range(args.n_epochs): for i in range(n_train_examples // args.batch_size): # Get the trainig batch X_batch, y_batch = next_batch(x_train, y_train, args.batch_size) # Reshape training batch, as explained above, in the shape of the placeholder X_batch = X_batch.reshape((args.batch_size, 1, args.n_inputs)) sess.run(training_op, feed_dict={ x_input: X_batch, target: y_batch }) # Obtain accuracy on training & test data acc_train = accuracy.eval(feed_dict={ x_input: X_batch, target: y_batch }) acc_test = accuracy.eval(feed_dict={ x_input: x_test, target: y_test }) # Update the best accuracy & output accuracy results if acc_test > best_accuracy: best_accuracy = acc_test last_improvement = epoch print("Epoch", epoch, ":\tTrain accuracy =", acc_train, ";\tTest accuracy =", acc_test) # Early stopping in case of no improvement. if epoch - last_improvement > check_improvement: print("\n#### No improvement seen in ", check_improvement, " epochs ==> early stopping.\n") break
def run_training(): if not os.path.exists(os.path.dirname(FLAGS.checkpoints_dir)): os.mkdir(os.path.dirname(FLAGS.checkpoints_dir)) if not os.path.exists(FLAGS.checkpoints_dir): os.mkdir(FLAGS.checkpoints_dir) # 单词转化的数字:向量,单词和数字一一对应的字典,单词 poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path) # 真实值和目标值 batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int) # 数据占位符 input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(vocabularies), rnn_size=128, num_layers=2, batch_size=64, learning_rate=FLAGS.learning_rate) # 实例化保存模型 saver = tf.train.Saver(tf.global_variables()) # 全局变量进行初始化 init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess) # sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) # 先执行,全局变量初始化 sess.run(init_op) start_epoch = 0 # 把之前训练过的checkpoint拿出来 checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoints_dir) if checkpoint: # 拿出训练保存模型 saver.restore(sess, checkpoint) print("[INFO] restore from the checkpoint {0}".format(checkpoint)) start_epoch += int(checkpoint.split('-')[-1]) print('[INFO] start training...') try: for epoch in range(start_epoch, FLAGS.epochs): n = 0 # 多少行唐诗//每次训练的个数 n_chunk = len(poems_vector) // FLAGS.batch_size for batch in range(n_chunk): loss, _, _ = sess.run( [ end_points['total_loss'], # 损失 end_points['last_state'], # 最后一次输出 end_points['train_op'] # 训练优化损失 ], feed_dict={ input_data: batches_inputs[n], output_targets: batches_outputs[n] }) n += 1 print( '[INFO] Epoch: %d , batch: %d , training loss: %.6f' % (epoch, batch, loss)) if epoch % 6 == 0: # 每隔多少次保存 saver.save(sess, FLAGS.checkpoints_dir, global_step=epoch) except KeyboardInterrupt: print( '[INFO] Interrupt manually, try saving checkpoint for now...') saver.save(sess, FLAGS.checkpoints_dir, global_step=epoch) print( '[INFO] Last epoch were saved, next time will start from epoch {}.' .format(epoch))
def gen_poem(begin_words, num): batch_size = 1 print('[INFO] loading corpus from %s' % FLAGS.file_path) # 单词转化的数字:向量,单词和数字一一对应的字典,单词 poems_vector, word_int_map, vocabularies = process_poems(FLAGS.file_path) # 此时输入为1个 input_data = tf.placeholder(tf.int32, [batch_size, None]) # 损失等 end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len(vocabularies), rnn_size=128, num_layers=2, batch_size=64, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) # 保存模型的位置,拿回sess checkpoint = tf.train.latest_checkpoint(FLAGS.checkpoints_dir) # checkpoint = tf.train.latest_checkpoint('./model/') saver.restore(sess, checkpoint) # saver.restore(sess,'./model/-24') # 从字典里面获取到的开始值 x = np.array([list(map(word_int_map.get, start_token))]) [predict, last_state ] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x}) poem = '' for begin_word in begin_words: while True: if begin_word: word = begin_word else: word = to_word(predict, vocabularies) sentence = '' while word != end_token: sentence += word x = np.zeros((1, 1)) x[0, 0] = word_int_map[word] [predict, last_state] = sess.run( [end_points['prediction'], end_points['last_state']], feed_dict={ input_data: x, end_points['initial_state']: last_state }) word = to_word(predict, vocabularies) # word = words[np.argmax(probs_)] if len(sentence) == 2 + 2 * num and (',' or '?') not in sentence[:num] and (',' or '?') not in sentence[ num + 1:-1] and \ sentence[num] == ',' and '□' not in sentence: poem += sentence # sentence = '' break else: print("我正在写诗呢") return poem
def run_training(FLAGS): if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) # 产生诗向量,字映射表,字集合 # pems_vector:[[2, 50, 179, 394, 1081, 597, 0, 13, 351, ...], [2, 161, 343, 973, 30, 2455, 0, 155, 297, ...], [2, 10, 89, 13, 644, 1085, 0, 340, 33, ...], [2, 644, 1081, 689, 204, 1116, 0, 25, 1242, ...], [2, 50, 259, 2401, 58, 137, 0, 25, 163, ...], [2, 6, 346, 25, 87, 106, 0, 439, 1926, ...], [2, 2951, 263, 1917, 28, 437, 0, 1155, 219, ...], [2, 238, 173, 873, 847, 552, 0, 1680, 175, ...], [2, 609, 1363, 17, 155, 56, 0, 915, 839, ...], [2, 202, 650, 79, 685, 3330, 0, 780, 539, ...], [2, 13, 4887, 1198, 2139, 699, 0, 1150, 2553, ...], [2, 107, 2126, 1745, 1103, 16, 0, 202, 1399, ...], [2, 89, 119, 296, 768, 173, 0, 200, 187, ...], [2, 539, 764, 630, 17, 386, 0, 435, 816, ...], ...] # word_to_int:{' ': 6109, '2': 5534, 'B': 2, 'E': 3, 'F': 5355, 'p': 5354, 'ē': 5531, 'ń': 5584, '□': 705, '、': 4110, '。': 1, '】': 5023, '一': 10, '丁': 1484, ...} # word:(',', '。', 'B', 'E', '不', '人', '山', '风', '日', '无', '一', '云', '花', '春', ...) poems_vector, word_to_int, words = process_poems(FLAGS.file_path) # baches_inputs:[array([[ 2, 50, ...ype=int32), array([[ 2, 75, ...ype=int32), array([[ 2, 53, ...ype=int32), array([[ 2, 121, ...ype=int32), array([[ 2, 357, ...ype=int32), array([[ 2, 168, ...ype=int32), array([[ 2, 2483, ...ype=int32), array([[ 2, 2501, ...ype=int32), array([[ 2, 283, ...ype=int32), array([[ 2, 80, ...ype=int32), array([[ 2, 2204, ...ype=int32), array([[ 2, 929, ...ype=int32), array([[ 2, 671, ...ype=int32), array([[ 2, 1440, ...ype=int32), ...] # batches_outputs:[array([[ 50, 179, ...ype=int32), array([[ 75, 35, ...ype=int32), array([[ 53, 963, ...ype=int32), array([[ 121, 328, ...ype=int32), array([[ 357, 283, ...ype=int32), array([[ 168, 21, ...ype=int32), array([[2483, 641, ...ype=int32), array([[2501, 2176, ...ype=int32), array([[ 283, 11, ...ype=int32), array([[ 80, 846, ...ype=int32), array([[2204, 2024, ...ype=int32), array([[ 929, 2500, ...ype=int32), array([[ 671, 1429, ...ype=int32), array([[1440, 211, ...ype=int32), ...] batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int) # inpuy_data:<tf.Tensor 'Placeholder:0' shape=(64, ?) dtype=int32> # output_targets:<tf.Tensor 'Placeholder_1:0' shape=(64, ?) dtype=int32> input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) # 构建模型 # end_points:{'initial_state': (LSTMStateTuple(c=<tf...=float32>), LSTMStateTuple(c=<tf...=float32>)), 'last_state': (LSTMStateTuple(c=<tf...=float32>), LSTMStateTuple(c=<tf...=float32>)), 'loss': <tf.Tensor 'softmax_...e=float32>, 'output': <tf.Tensor 'Reshape:...e=float32>, 'total_loss': <tf.Tensor 'Mean:0' ...e=float32>, 'train_op': <tf.Operation 'Adam'...type=NoOp>} end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(words), rnn_size=FLAGS.rnn_size, num_layers=FLAGS.num_layers, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate) # tf里面提供模型保存的是tf.train.Saver()模块。检查checkpoint 内容最好的方法是使用Saver 加载它。 # saver:<tensorflow.python.training.saver.Saver object at 0x7fc7756eaf28> saver = tf.train.Saver(tf.global_variables()) # 创建一个包含几个操作的op结点,当这个op结点运行完成,所有作为input的ops都被运行完成,这个操作没有返回值 # inti_op:<tf.Operation 'group_deps' type=NoOp> init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # 写入tensorflow日志 writer = tf.summary.FileWriter(FLAGS.tensorflow_logs, sess.graph) # 运行以上的节点 sess.run(init_op) start_epoch = 0 checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) # 从上次中断的checkpoint开始训练 if checkpoint: saver.restore(sess, checkpoint) print("## restore from the checkpoint {0}".format(checkpoint)) start_epoch += int(checkpoint.split('-')[-1]) print('## start training...') file = open(FLAGS.log_path, 'a') try: for epoch in range(start_epoch, FLAGS.epochs): n = 0 # 输入的数据集切成多少块 # n_chunk = len(poems_vector) // FLAGS.batch_size n_chunk = 50 for batch in range(n_chunk): loss, _, _ = sess.run( [ end_points['total_loss'], end_points['last_state'], end_points[ 'train_op'] # batches_inputs[n]代表取出第几块数据集 ], feed_dict={ input_data: batches_inputs[n], output_targets: batches_outputs[n] }) n += 1 print('Epoch: %d, batch: %d, training loss: %.6f' % (epoch, batch, loss)) file.write('Epoch: %d, batch: %d, training loss: %.6f' % (epoch, batch, loss) + "\n") if epoch % 6 == 0: saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) except KeyboardInterrupt: # 如果Ctrl+c中断,保存checkpoint, print('## Interrupt manually, try saving checkpoint for now...') saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) print( '## Last epoch were saved, next time will start from epoch {}.' .format(epoch)) writer.close() file.close() file.close() writer.close()
def train(dataLoader, validate_after=5, resume=False, perform_training=True, save_best=False, model_='cnn'): """ Perform training and validation of model. Args: dataLoader : DataLoader object validate_after : Number of epochs after which validation is performed. The model is also saved after this. resume : If True, a previously saved model file is loaded. perform_training : If False, training step is skipped, and final testing is done. save_best : If True, save session for epoch with minimum validation loss. model_ : String denoting the neural network model to use (RNN or CNN) """ model = None if model_ == 'cnn': model = models.cnn_model() elif model_ == 'rnn': model = models.rnn_model() sess = tf.Session() saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) if resume: try: prev_session = config.resume_ckpt saver.restore(sess, prev_session) print("Using previous session: {}".format(prev_session)) except Exception as exp: print(exp) print("Creating a new session.") if save_best: MIN_VAL_LOSS = 100000000000 if perform_training: config.init() train_writer = tf.summary.FileWriter( os.path.join(config.logdir, "train"), sess.graph) valid_writer = tf.summary.FileWriter( os.path.join(config.logdir, "validation"), sess.graph) for e in range(config.EPOCHS): epoch_loss = 0.0 for sensor, label in dataLoader.next_train(): # Run the graph. loss, _, tb = sess.run( [model['loss'], model['train'], model['summary']], feed_dict={ model['sensor_data']: sensor, model['label']: label, model['training']: True }) epoch_loss += loss avg_loss = epoch_loss / dataLoader.train_batches print("Average loss for epoch {} = {}".format(e, avg_loss)) if e % validate_after == 0: val_loss = validation(sess, model, dataLoader, valid_writer, e) if save_best: if val_loss < MIN_VAL_LOSS: path = saver.save(sess, config.ckpt, global_step=e) print("Saved model to {}".format(path)) MIN_VAL_LOSS = val_loss else: path = saver.save(sess, config.ckpt, global_step=e) print("Saved model to {}".format(path)) train_writer.add_summary(tb, e) print("===========================================") print("Calculating validation accuracy...") accuracies = [] positives = negatives = 0 true_positives = true_negatives = false_positives = false_negatives = 0 for sensor, label in dataLoader.next_validation(): # Run the graph. pred = sess.run(model['prediction'], feed_dict={ model['sensor_data']: sensor, model['label']: label, model['training']: False }) label = np.argmax(label, axis=1) positives += np.count_nonzero(label == 1) negatives += np.count_nonzero(label == 0) # detects the condition when the condition is present. true_positives += np.count_nonzero(pred + label == 2) # does not detect the condition when the condition is absent. true_negatives += np.count_nonzero(pred + label == 0) # wrongly indicates that a particular condition or attribute is present. false_positives += np.count_nonzero(pred > label) # wrongly indicates that a particular condition or attribute is absent. false_negatives += np.count_nonzero(pred < label) accuracies.append( np.count_nonzero(pred == label) / pred.shape[0] * 100) accuracies = np.array(accuracies) # print(positives, negatives) # print("True positives : {}".format(true_positives)) # print("False negatives: {}".format(false_negatives)) # print("False positives: {}".format(false_positives)) # print("True negatives: {}".format(true_negatives)) print("Sensitivity: {}".format(true_positives / positives)) print("Specificity: {}".format(true_negatives / negatives)) print("Precision: {}".format(true_positives / (true_positives + false_positives))) print("Min Validation set accuracy: {} %".format(accuracies.min())) print("Max Validation set accuracy: {} %".format(accuracies.max())) print("Average Validation set accuracy: {} %".format(accuracies.mean())) sess.close()
def run_training(): if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path) batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int) input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(vocabularies), rnn_size=128, num_layers=2, batch_size=64, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess) # sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) sess.run(init_op) start_epoch = 0 checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) if checkpoint: saver.restore(sess, checkpoint) print("## restore from the checkpoint {0}".format(checkpoint)) start_epoch += int(checkpoint.split('-')[-1]) print('## start training...') try: for epoch in range(start_epoch, FLAGS.epochs): n = 0 n_chunk = len(poems_vector) // FLAGS.batch_size for batch in range(n_chunk): loss, _, _ = sess.run( [ end_points['total_loss'], end_points['last_state'], end_points['train_op'] ], feed_dict={ input_data: batches_inputs[n], output_targets: batches_outputs[n] }) n += 1 print('Epoch: %d, batch: %d, training loss: %.6f' % (epoch, batch, loss)) if epoch % 6 == 0: saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) except KeyboardInterrupt: print('## Interrupt manually, try saving checkpoint for now...') saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) print( '## Last epoch were saved, next time will start from epoch {}.' .format(epoch))
def run_training(): # 模型保存路径不存在则创建 if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) # process_poems对古诗进行预处理 poems_vector, word_to_int, vocabularies = process_poems(FLAGS.file_path) # batches_inputs, batches_outputs = generate_batch(FLAGS.batch_size, poems_vector, word_to_int) # 占位向量 input_data = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) output_targets = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) # 使用lstm模型进行训练 end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(vocabularies), rnn_size=128, num_layers=2, batch_size=FLAGS.batch_size, learning_rate=FLAGS.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # sess = tf_debug.LocalCLIDebugWrapperSession(sess=sess) # sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) sess.run(init_op) start_epoch = 0 checkpoint = tf.train.latest_checkpoint(FLAGS.model_dir) # 如果之前训练过就找回之前的训练结果 if checkpoint: saver.restore(sess, checkpoint) print("## restore from the checkpoint {0}".format(checkpoint)) start_epoch += int(checkpoint.split('-')[-1]) print('## start training...') try: n_chunk = len(poems_vector) // FLAGS.batch_size for epoch in range(start_epoch, FLAGS.epochs): #每次对其中的数据shuffle一次 # print(type(poems_vector)) random.shuffle(poems_vector) # print(type(poems_vector)) # 这里有每一次输入的训练数据的列表 batches_inputs, batches_outputs = generate_batch( FLAGS.batch_size, poems_vector, word_to_int) n = 0 for batch in range(n_chunk): loss, _, _ = sess.run( [ end_points['total_loss'], end_points['last_state'], end_points['train_op'] ], feed_dict={ input_data: batches_inputs[n], output_targets: batches_outputs[n] }) n += 1 print('Epoch: %d/%d, batch: %d/%d, training loss: %.6f' % (epoch, FLAGS.epochs - 1, batch, n_chunk - 1, loss)) if epoch % 6 == 0: saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) alltime = time.time() - start_time print("Time: %d h %d min % ds" % (alltime // 3600, (alltime - alltime // 3600 * 3600) // 60, alltime % 60)) except KeyboardInterrupt: print('## Interrupt manually, try saving checkpoint for now...') saver.save(sess, os.path.join(FLAGS.model_dir, FLAGS.model_prefix), global_step=epoch) print( '## Last epoch were saved, next time will start from epoch {}.' .format(epoch))
def gen_poem(begin_word): batch_size = 1 print('## loading corpus from %s' % model_dir) poems_vector, word_int_map, vocabularies = process_poems(corpus_file) input_data = tf.placeholder(tf.int32, [batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len(vocabularies), rnn_size=128, num_layers=2, batch_size=64, learning_rate=lr) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) checkpoint = tf.train.latest_checkpoint(model_dir) # print(checkpoint) saver.restore(sess, checkpoint) x = np.array([list(map(word_int_map.get, start_token))]) [predict, last_state ] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x}) poem_ = '' word = begin_word or to_word(predict, vocabularies, poem=poem_) i = 0 # while not poem_ or ' ' in poem_: if 1: print("开始作诗") poem_ = '' while word != end_token and word != start_token: # while 1: # if word == ' ': # poem_ = gen_poem(begin_word) # break poem_ += word print(poem_) i += 1 # if i > 24: # break x = np.array([[word_int_map[word]]]) [predict, last_state] = sess.run( [end_points['prediction'], end_points['last_state']], feed_dict={ input_data: x, end_points['initial_state']: last_state }) word = ' ' # while word == ' ': if 1: time.sleep(1) random.seed(time.time()) word = to_word(predict, vocabularies, poem_) print('\n') return poem_