def main(_): #1.load data(X:list of lint,y:int). #if os.path.exists(FLAGS.cache_path): # load training data from cache file. # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) #else: if 1==1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary(word2vec_model_path=FLAGS.word2vec_model_path,name_scope="dynamic_memory_network") #simple='simple' vocab_size = len(vocabulary_word2index) print("dynamic_memory_network.vocab_size:",vocab_size) vocabulary_word2index_label,vocabulary_index2word_label = create_voabulary_label(name_scope="dynamic_memory_network") if FLAGS.multi_label_flag: FLAGS.traning_data_path='../training-data/train-zhihu6-title-desc.txt' #change this line if want to train in a small dataset. e.g. dataset from 'test-zhihu6-title-desc.txt' train,test,_=load_data_multilabel_new(vocabulary_word2index,vocabulary_word2index_label,multi_label_flag=FLAGS.multi_label_flag, traning_data_path=FLAGS.traning_data_path) trainX, trainY = train testX, testY = test print("trainY:",trainY[0:10]) # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length #with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) #;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") #2.create session. config=tf.ConfigProto() config.gpu_options.allow_growth=True with tf.Session(config=config) as sess: #Instantiate Model model = DynamicMemoryNetwork(FLAGS.num_classes, FLAGS.learning_rate, FLAGS.batch_size, FLAGS.decay_steps, FLAGS.decay_rate, FLAGS.sequence_length, FLAGS.story_length,vocab_size, FLAGS.embed_size, FLAGS.hidden_size, FLAGS.is_training,num_pass=FLAGS.num_pass, use_gated_gru=FLAGS.use_gated_gru,decode_with_sequences=FLAGS.decode_with_sequences,multi_label_flag=FLAGS.multi_label_flag,l2_lambda=FLAGS.l2_lambda) #Initialize Save saver=tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir+"checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess,tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: #load pre-trained word embedding assign_pretrained_word_embedding(sess, vocabulary_index2word, vocab_size, model,word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch=sess.run(model.epoch_step) #3.feed data & training number_of_training_data=len(trainX) print("number_of_training_data:",number_of_training_data) previous_eval_loss=10000 best_eval_loss=10000 batch_size=FLAGS.batch_size for epoch in range(curr_epoch,FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip(range(0, number_of_training_data, batch_size),range(batch_size, number_of_training_data, batch_size)): if epoch==0 and counter==0: print("trainX[start:end]:",trainX[start:end])#;print("trainY[start:end]:",trainY[start:end]) feed_dict = {model.query: trainX[start:end],model.story: np.expand_dims(trainX[start:end],axis=1),model.dropout_keep_prob: 1.0} if not FLAGS.multi_label_flag: feed_dict[model.answer_single] = trainY[start:end] else: feed_dict[model.answer_multilabel]=trainY[start:end] curr_loss,curr_acc,_=sess.run([model.loss_val,model.accuracy,model.train_op],feed_dict) #curr_acc--->TextCNN.accuracy loss,counter,acc=loss+curr_loss,counter+1,acc+curr_acc if counter %50==0: print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" %(epoch,counter,math.exp(loss/float(counter)) if (loss/float(counter))<20 else 10000.000,acc/float(counter))) #tTrain Accuracy:%.3f---》acc/float(counter) ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### if FLAGS.batch_size!=0 and (start%(FLAGS.validate_step*FLAGS.batch_size)==0): #(epoch % FLAGS.validate_every) or if epoch % FLAGS.validate_every == 0: eval_loss, eval_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) print("dynamic_memory_network[use_gated_gru=False,num_pass=2].validation.part. previous_eval_loss:", math.exp(previous_eval_loss) if previous_eval_loss<20 else 10000.000,";current_eval_loss:", math.exp(eval_loss) if eval_loss<20 else 10000.000) if eval_loss > previous_eval_loss: #if loss is not decreasing # reduce the learning rate by a factor of 0.5 print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>validation.part.going to reduce the learning rate.") learning_rate1 = sess.run(model.learning_rate) lrr=sess.run([model.learning_rate_decay_half_op]) learning_rate2 = sess.run(model.learning_rate) print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>validation.part.learning_rate1:", learning_rate1, " ;learning_rate2:",learning_rate2) else:# loss is decreasing if eval_loss<best_eval_loss: print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>going to save the model.eval_loss:",math.exp(eval_loss) if eval_loss<20 else 10000.000,";best_eval_loss:",math.exp(best_eval_loss) if best_eval_loss<20 else 10000.000) # save model to checkpoint save_path = FLAGS.ckpt_dir + "model.ckpt" saver.save(sess, save_path, global_step=epoch) best_eval_loss=eval_loss previous_eval_loss = eval_loss ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### #epoch increment print("going to increment epoch counter....") sess.run(model.epoch_increment) # 5.test on test set test_loss, test_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) pass
def main(_): #1.load data(X:list of lint,y:int). #if os.path.exists(FLAGS.cache_path): # 如果文件系统中存在,那么加载故事(词汇表索引化的) # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) #else: if 1 == 1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary( word2vec_model_path=FLAGS.word2vec_model_path, name_scope="cnn2") #simple='simple' vocab_size = len(vocabulary_word2index) print("cnn_model.vocab_size:", vocab_size) vocabulary_word2index_label, vocabulary_index2word_label = create_voabulary_label( name_scope="cnn2") if FLAGS.multi_label_flag: FLAGS.traning_data_path = 'training-data/train-zhihu6-title-desc.txt' #test-zhihu5-only-title-multilabel.txt train, test, _ = load_data_multilabel_new( vocabulary_word2index, vocabulary_word2index_label, multi_label_flag=FLAGS.multi_label_flag, traning_data_path=FLAGS.traning_data_path ) #,traning_data_path=FLAGS.traning_data_path trainX, trainY = train testX, testY = test # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sentence_len, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sentence_len, value=0.) # padding to max length #with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) #;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") #2.create session. config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: #Instantiate Model textCNN = TextCNN(filter_sizes, FLAGS.num_filters, FLAGS.num_classes, FLAGS.learning_rate, FLAGS.batch_size, FLAGS.decay_steps, FLAGS.decay_rate, FLAGS.sentence_len, vocab_size, FLAGS.embed_size, FLAGS.is_training, multi_label_flag=FLAGS.multi_label_flag) #Initialize Save saver = tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir + "checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: #load pre-trained word embedding assign_pretrained_word_embedding( sess, vocabulary_index2word, vocab_size, textCNN, word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch = sess.run(textCNN.epoch_step) #3.feed data & training number_of_training_data = len(trainX) batch_size = FLAGS.batch_size for epoch in range(curr_epoch, FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip( range(0, number_of_training_data, batch_size), range(batch_size, number_of_training_data, batch_size)): if epoch == 0 and counter == 0: print("trainX[start:end]:", trainX[start:end] ) #;print("trainY[start:end]:",trainY[start:end]) feed_dict = { textCNN.input_x: trainX[start:end], textCNN.dropout_keep_prob: 0.5 } if not FLAGS.multi_label_flag: feed_dict[textCNN.input_y] = trainY[start:end] else: feed_dict[textCNN.input_y_multilabel] = trainY[start:end] curr_loss, curr_acc, _ = sess.run( [textCNN.loss_val, textCNN.accuracy, textCNN.train_op], feed_dict) #curr_acc--->TextCNN.accuracy loss, counter, acc = loss + curr_loss, counter + 1, acc + curr_acc if counter % 50 == 0: print( "Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" % (epoch, counter, loss / float(counter), acc / float(counter)) ) #tTrain Accuracy:%.3f---》acc/float(counter) #epoch increment print("going to increment epoch counter....") sess.run(textCNN.epoch_increment) # 4.validation print(epoch, FLAGS.validate_every, (epoch % FLAGS.validate_every == 0)) if epoch % FLAGS.validate_every == 0: eval_loss, eval_acc = do_eval(sess, textCNN, testX, testY, batch_size, vocabulary_index2word_label) print( "Epoch %d Validation Loss:%.3f\tValidation Accuracy: %.3f" % (epoch, eval_loss, eval_acc)) #save model to checkpoint save_path = FLAGS.ckpt_dir + "model.ckpt" saver.save(sess, save_path, global_step=epoch) # 5.最后在测试集上做测试,并报告测试准确率 Test test_loss, test_acc = do_eval(sess, textCNN, testX, testY, batch_size, vocabulary_index2word_label) pass
def main(_): #1.load data(X:list of lint,y:int). #if os.path.exists(FLAGS.cache_path): # 如果文件系统中存在,那么加载故事(词汇表索引化的) # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) #else: if 1==1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary(word2vec_model_path=FLAGS.word2vec_model_path,name_scope="rcnn") #simple='simple' vocab_size = len(vocabulary_word2index) print("cnn_model.vocab_size:",vocab_size) vocabulary_word2index_label,vocabulary_index2word_label = create_voabulary_label(name_scope="rcnn") if FLAGS.multi_label_flag: FLAGS.traning_data_path='training-data/train-zhihu6-title-desc.txt' #test-zhihu5-only-title-multilabel.txt train, test, _ = load_data_multilabel_new(vocabulary_word2index, vocabulary_word2index_label,multi_label_flag=FLAGS.multi_label_flag,traning_data_path=FLAGS.traning_data_path) #,traning_data_path=FLAGS.traning_data_path trainX, trainY = train testX, testY = test # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length #with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) #;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") #2.create session. config=tf.ConfigProto() config.gpu_options.allow_growth=True with tf.Session(config=config) as sess: #Instantiate Model textRCNN=TextRCNN(FLAGS.num_classes, FLAGS.learning_rate, FLAGS.decay_steps, FLAGS.decay_rate,FLAGS.sequence_length, vocab_size,FLAGS.embed_size,FLAGS.is_training,FLAGS.batch_size,multi_label_flag=FLAGS.multi_label_flag) #Initialize Save saver=tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir+"checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess,tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: #load pre-trained word embedding assign_pretrained_word_embedding(sess, vocabulary_index2word, vocab_size, textRCNN,word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch=sess.run(textRCNN.epoch_step) #3.feed data & training number_of_training_data=len(trainX) batch_size=FLAGS.batch_size for epoch in range(curr_epoch,FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip(range(0, number_of_training_data, batch_size),range(batch_size, number_of_training_data, batch_size)): if epoch==0 and counter==0: print("trainX[start:end]:",trainX[start:end])#;print("trainY[start:end]:",trainY[start:end]) feed_dict = {textRCNN.input_x: trainX[start:end],textRCNN.dropout_keep_prob: 0.5} if not FLAGS.multi_label_flag: feed_dict[textRCNN.input_y] = trainY[start:end] else: feed_dict[textRCNN.input_y_multilabel]=trainY[start:end] curr_loss,curr_acc,_=sess.run([textRCNN.loss_val,textRCNN.accuracy,textRCNN.train_op],feed_dict) #curr_acc--->TextCNN.accuracy loss,counter,acc=loss+curr_loss,counter+1,acc+curr_acc if counter %50==0: print("Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" %(epoch,counter,loss/float(counter),acc/float(counter))) #tTrain Accuracy:%.3f---》acc/float(counter) #epoch increment print("going to increment epoch counter....") sess.run(textRCNN.epoch_increment) # 4.validation print(epoch,FLAGS.validate_every,(epoch % FLAGS.validate_every==0)) if epoch % FLAGS.validate_every==0: eval_loss, eval_acc=do_eval(sess,textRCNN,testX,testY,batch_size,vocabulary_index2word_label) print("Epoch %d Validation Loss:%.3f\tValidation Accuracy: %.3f" % (epoch,eval_loss,eval_acc)) #save model to checkpoint save_path=FLAGS.ckpt_dir+"model.ckpt" saver.save(sess,save_path,global_step=epoch) # 5.最后在测试集上做测试,并报告测试准确率 Test test_loss, test_acc = do_eval(sess, textRCNN, testX, testY, batch_size,vocabulary_index2word_label) pass
def main(_): # 1.load data(X:list of lint,y:int). # if os.path.exists(FLAGS.cache_path): # 如果文件系统中存在,那么加载故事(词汇表索引化的) # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) # else: if 1 == 1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary( word2vec_model_path=FLAGS.word2vec_model_path, name_scope="transformer_classification") # simple='simple' vocab_size = len(vocabulary_word2index) print("transformer.vocab_size:", vocab_size) vocabulary_word2index_label, vocabulary_index2word_label = create_voabulary_label( name_scope="transformer_classification") if FLAGS.multi_label_flag: FLAGS.traning_data_path = 'training-data/test-zhihu6-title-desc.txt' # one record like this:'w35620 w1097 w111 c278 c150 c150 c285 c278 c43 __label__7756633728210171144 3195914392210930723' train, test, _ = load_data_multilabel_new( vocabulary_word2index, vocabulary_word2index_label, multi_label_flag=FLAGS.multi_label_flag, traning_data_path=FLAGS.traning_data_path) trainX, trainY, = train testX, testY = test print("trainY:", trainY[0:10]) # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length # with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) # ;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") # 2.create session. config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: # Instantiate Model model = Transformer(FLAGS.num_classes, FLAGS.learning_rate, FLAGS.batch_size, FLAGS.decay_steps, FLAGS.decay_rate, FLAGS.sequence_length, vocab_size, FLAGS.embed_size, FLAGS.d_model, FLAGS.d_k, FLAGS.d_v, FLAGS.h, FLAGS.num_layer, FLAGS.is_training, l2_lambda=FLAGS.l2_lambda) # Initialize Save saver = tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir + "checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: # load pre-trained word embedding assign_pretrained_word_embedding( sess, vocabulary_index2word, vocab_size, model, word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch = sess.run(model.epoch_step) # 3.feed data & training number_of_training_data = len(trainX) print("number_of_training_data:", number_of_training_data) previous_eval_loss = 10000 best_eval_loss = 10000 batch_size = FLAGS.batch_size for epoch in range(curr_epoch, FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip( range(0, number_of_training_data, batch_size), range(batch_size, number_of_training_data, batch_size)): if epoch == 0 and counter == 0: print("trainX[start:end]:", trainX[start:end]) feed_dict = { model.input_x: trainX[start:end], model.dropout_keep_prob: 0.5 } feed_dict[model.input_y_label] = trainY[start:end] curr_loss, curr_acc, _ = sess.run( [model.loss_val, model.accuracy, model.train_op], feed_dict) # curr_acc--->TextCNN.accuracy loss, counter, acc = loss + curr_loss, counter + 1, acc + curr_acc if counter % 50 == 0: print( "transformer.classification==>Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" % (epoch, counter, loss / float(counter), acc / float(counter)) ) # tTrain Accuracy:%.3f---》acc/float(counter) ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### if FLAGS.batch_size != 0 and ( start % (FLAGS.validate_step * FLAGS.batch_size) == 0): eval_loss, eval_acc = do_eval(sess, model, testX, testY, batch_size, vocabulary_index2word_label) print( "transformer.classification.validation.part. previous_eval_loss:", previous_eval_loss, ";current_eval_loss:", eval_loss) if eval_loss > previous_eval_loss: # if loss is not decreasing # reduce the learning rate by a factor of 0.5 print( "transformer.classification.==>validation.part.going to reduce the learning rate." ) learning_rate1 = sess.run(model.learning_rate) lrr = sess.run([model.learning_rate_decay_half_op]) learning_rate2 = sess.run(model.learning_rate) print( "transformer.classification==>validation.part.learning_rate1:", learning_rate1, " ;learning_rate2:", learning_rate2) # print("HierAtten==>Epoch %d Validation Loss:%.3f\tValidation Accuracy: %.3f" % (epoch, eval_loss, eval_acc)) else: # loss is decreasing if eval_loss < best_eval_loss: print( "transformer.classification==>going to save the model.eval_loss:", eval_loss, ";best_eval_loss:", best_eval_loss) # save model to checkpoint save_path = FLAGS.ckpt_dir + "model.ckpt" saver.save(sess, save_path, global_step=epoch) best_eval_loss = eval_loss previous_eval_loss = eval_loss ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### # epoch increment print("going to increment epoch counter....") sess.run(model.epoch_increment) # 5.最后在测试集上做测试,并报告测试准确率 Test test_loss, test_acc = do_eval(sess, model, testX, testY, batch_size, vocabulary_index2word_label) pass
def main(_): #1.load data(X:list of lint,y:int). #if os.path.exists(FLAGS.cache_path): # load training data from cache file. # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) #else: if 1==1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary(word2vec_model_path=FLAGS.word2vec_model_path,name_scope="dynamic_memory_network") #simple='simple' vocab_size = len(vocabulary_word2index) print("dynamic_memory_network.vocab_size:",vocab_size) vocabulary_word2index_label,vocabulary_index2word_label = create_voabulary_label(name_scope="dynamic_memory_network") if FLAGS.multi_label_flag: FLAGS.traning_data_path='../training-data/train-zhihu6-title-desc.txt' #change this line if want to train in a small dataset. e.g. dataset from 'test-zhihu6-title-desc.txt' train,test,_=load_data_multilabel_new(vocabulary_word2index,vocabulary_word2index_label,multi_label_flag=FLAGS.multi_label_flag, traning_data_path=FLAGS.traning_data_path) trainX, trainY = train testX, testY = test print("trainY:",trainY[0:10]) # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length #with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) #;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") #2.create session. config=tf.ConfigProto() config.gpu_options.allow_growth=True with tf.Session(config=config) as sess: #Instantiate Model model = DynamicMemoryNetwork(FLAGS.num_classes, FLAGS.learning_rate, FLAGS.batch_size, FLAGS.decay_steps, FLAGS.decay_rate, FLAGS.sequence_length, FLAGS.story_length,vocab_size, FLAGS.embed_size, FLAGS.hidden_size, FLAGS.is_training,num_pass=FLAGS.num_pass, use_gated_gru=FLAGS.use_gated_gru,decode_with_sequences=FLAGS.decode_with_sequences,multi_label_flag=FLAGS.multi_label_flag,l2_lambda=FLAGS.l2_lambda) #Initialize Save saver=tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir+"checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess,tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: #load pre-trained word embedding assign_pretrained_word_embedding(sess, vocabulary_index2word, vocab_size, model,word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch=sess.run(model.epoch_step) #3.feed data & training number_of_training_data=len(trainX) print("number_of_training_data:",number_of_training_data) previous_eval_loss=10000 best_eval_loss=10000 batch_size=FLAGS.batch_size for epoch in range(curr_epoch,FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip(range(0, number_of_training_data, batch_size),range(batch_size, number_of_training_data, batch_size)): if epoch==0 and counter==0: print("trainX[start:end]:",trainX[start:end])#;print("trainY[start:end]:",trainY[start:end]) feed_dict = {model.query: trainX[start:end],model.story: np.expand_dims(trainX[start:end],axis=1),model.dropout_keep_prob: 1.0} if not FLAGS.multi_label_flag: feed_dict[model.answer_single] = trainY[start:end] else: feed_dict[model.answer_multilabel]=trainY[start:end] curr_loss,curr_acc,_=sess.run([model.loss_val,model.accuracy,model.train_op],feed_dict) #curr_acc--->TextCNN.accuracy loss,counter,acc=loss+curr_loss,counter+1,acc+curr_acc if counter %50==0: print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" %(epoch,counter,math.exp(loss/float(counter)) if (loss/float(counter))<20 else 10000.000,acc/float(counter))) #tTrain Accuracy:%.3f---》acc/float(counter) ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### if FLAGS.batch_size!=0 and (start%(FLAGS.validate_step*FLAGS.batch_size)==0): #(epoch % FLAGS.validate_every) or if epoch % FLAGS.validate_every == 0: eval_loss, eval_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) print("dynamic_memory_network[use_gated_gru=False,num_pass=2].validation.part. previous_eval_loss:", math.exp(previous_eval_loss) if previous_eval_loss<20 else 10000.000,";current_eval_loss:", math.exp(eval_loss) if eval_loss<20 else 10000.000) if eval_loss > previous_eval_loss: #if loss is not decreasing # reduce the learning rate by a factor of 0.5 print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>validation.part.going to reduce the learning rate.") learning_rate1 = sess.run(model.learning_rate) lrr=sess.run([model.learning_rate_decay_half_op]) learning_rate2 = sess.run(model.learning_rate) print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>validation.part.learning_rate1:", learning_rate1, " ;learning_rate2:",learning_rate2) else:# loss is decreasing if eval_loss<best_eval_loss: print("dynamic_memory_network[use_gated_gru=False,num_pass=2]==>going to save the model.eval_loss:",math.exp(eval_loss) if eval_loss<20 else 10000.000,";best_eval_loss:",math.exp(best_eval_loss) if best_eval_loss<20 else 10000.000) # save model to checkpoint save_path = FLAGS.ckpt_dir + "model.ckpt" saver.save(sess, save_path, global_step=epoch) best_eval_loss=eval_loss previous_eval_loss = eval_loss ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### #epoch increment print("going to increment epoch counter....") sess.run(model.epoch_increment) # 5.test on test set test_loss, test_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) pass
def main(_): #1.load data(X:list of lint,y:int). #if os.path.exists(FLAGS.cache_path): # 如果文件系统中存在,那么加载故事(词汇表索引化的) # with open(FLAGS.cache_path, 'r') as data_f: # trainX, trainY, testX, testY, vocabulary_index2word=pickle.load(data_f) # vocab_size=len(vocabulary_index2word) #else: if 1==1: trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary(word2vec_model_path=FLAGS.word2vec_model_path,name_scope="transformer_classification") #simple='simple' vocab_size = len(vocabulary_word2index) print("transformer.vocab_size:",vocab_size) vocabulary_word2index_label,vocabulary_index2word_label = create_voabulary_label(name_scope="transformer_classification") if FLAGS.multi_label_flag: FLAGS.traning_data_path='training-data/test-zhihu6-title-desc.txt' #one record like this:'w35620 w1097 w111 c278 c150 c150 c285 c278 c43 __label__7756633728210171144 3195914392210930723' train,test,_=load_data_multilabel_new(vocabulary_word2index,vocabulary_word2index_label,multi_label_flag=FLAGS.multi_label_flag, traning_data_path=FLAGS.traning_data_path) trainX, trainY, = train testX, testY = test print("trainY:",trainY[0:10]) # 2.Data preprocessing.Sequence padding print("start padding & transform to one hot...") trainX = pad_sequences(trainX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length testX = pad_sequences(testX, maxlen=FLAGS.sequence_length, value=0.) # padding to max length #with open(FLAGS.cache_path, 'w') as data_f: #save data to cache file, so we can use it next time quickly. # pickle.dump((trainX,trainY,testX,testY,vocabulary_index2word),data_f) print("trainX[0]:", trainX[0]) #;print("trainY[0]:", trainY[0]) # Converting labels to binary vectors print("end padding & transform to one hot...") #2.create session. config=tf.ConfigProto() config.gpu_options.allow_growth=True with tf.Session(config=config) as sess: #Instantiate Model model=Transformer(FLAGS.num_classes, FLAGS.learning_rate, FLAGS.batch_size, FLAGS.decay_steps, FLAGS.decay_rate, FLAGS.sequence_length, vocab_size, FLAGS.embed_size,FLAGS.d_model,FLAGS.d_k,FLAGS.d_v,FLAGS.h,FLAGS.num_layer,FLAGS.is_training,l2_lambda=FLAGS.l2_lambda) #Initialize Save saver=tf.train.Saver() if os.path.exists(FLAGS.ckpt_dir+"checkpoint"): print("Restoring Variables from Checkpoint") saver.restore(sess,tf.train.latest_checkpoint(FLAGS.ckpt_dir)) else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if FLAGS.use_embedding: #load pre-trained word embedding assign_pretrained_word_embedding(sess, vocabulary_index2word, vocab_size, model,word2vec_model_path=FLAGS.word2vec_model_path) curr_epoch=sess.run(model.epoch_step) #3.feed data & training number_of_training_data=len(trainX) print("number_of_training_data:",number_of_training_data) previous_eval_loss=10000 best_eval_loss=10000 batch_size=FLAGS.batch_size for epoch in range(curr_epoch,FLAGS.num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip(range(0, number_of_training_data, batch_size),range(batch_size, number_of_training_data, batch_size)): if epoch==0 and counter==0: print("trainX[start:end]:",trainX[start:end]) feed_dict = {model.input_x: trainX[start:end],model.dropout_keep_prob: 0.5} feed_dict[model.input_y_label]=trainY[start:end] curr_loss,curr_acc,_=sess.run([model.loss_val,model.accuracy,model.train_op],feed_dict) #curr_acc--->TextCNN.accuracy loss,counter,acc=loss+curr_loss,counter+1,acc+curr_acc if counter %50==0: print("transformer.classification==>Epoch %d\tBatch %d\tTrain Loss:%.3f\tTrain Accuracy:%.3f" %(epoch,counter,loss/float(counter),acc/float(counter))) #tTrain Accuracy:%.3f---》acc/float(counter) ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### if FLAGS.batch_size!=0 and (start%(FLAGS.validate_step*FLAGS.batch_size)==0): eval_loss, eval_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) print("transformer.classification.validation.part. previous_eval_loss:", previous_eval_loss,";current_eval_loss:",eval_loss) if eval_loss > previous_eval_loss: #if loss is not decreasing # reduce the learning rate by a factor of 0.5 print("transformer.classification.==>validation.part.going to reduce the learning rate.") learning_rate1 = sess.run(model.learning_rate) lrr=sess.run([model.learning_rate_decay_half_op]) learning_rate2 = sess.run(model.learning_rate) print("transformer.classification==>validation.part.learning_rate1:", learning_rate1, " ;learning_rate2:",learning_rate2) #print("HierAtten==>Epoch %d Validation Loss:%.3f\tValidation Accuracy: %.3f" % (epoch, eval_loss, eval_acc)) else:# loss is decreasing if eval_loss<best_eval_loss: print("transformer.classification==>going to save the model.eval_loss:",eval_loss,";best_eval_loss:",best_eval_loss) # save model to checkpoint save_path = FLAGS.ckpt_dir + "model.ckpt" saver.save(sess, save_path, global_step=epoch) best_eval_loss=eval_loss previous_eval_loss = eval_loss ##VALIDATION VALIDATION VALIDATION PART###################################################################################################### #epoch increment print("going to increment epoch counter....") sess.run(model.epoch_increment) # 5.最后在测试集上做测试,并报告测试准确率 Test test_loss, test_acc = do_eval(sess, model, testX, testY, batch_size,vocabulary_index2word_label) pass
def main(_): # 1. load data if 1 == 1: # 1. get vocabulary of label. # trainX, trainY, testX, testY = None, None, None, None vocabulary_word2index, vocabulary_index2word = create_voabulary( simple='simple', word2vec_model_path=word2vec_model_path, name_scope='rnn') vocab_size = len(vocabulary_word2index) print('rnn_model.vocab_size:', vocab_size) vocabulary_word2index_label, vocabulary_index2word_label = create_voabulary_label( name_scope='rnn', voabulary_label=traing_data_path) train, test, _ = load_data_multilabel_new( vocabulary_word2index, vocabulary_word2index_label, multi_label_flag=False, traning_data_path=traing_data_path) trainX, trainY = train testX, testY = test # 2. data preprocessing.Sequence padding print('start padding & transform to one hot ...') trainX = pad_sequences(trainX, maxlen=sequence_length, value=0.0) # padding to max length testX = pad_sequences(testX, maxlen=sequence_length, value=0.0) print( 'trainX[0]:', trainX[0], ) # convert labels to binary vector print('end padding & transform to one hot ...') # 2. create session config = tf.ConfigProto() config.gpu_options.allow_growth = True # 动态的分配gpu空间,需要多少占用多少 with tf.Session(config=config) as sess: # instantiate mdoel textRNN = TextRNN(num_classes, learning_rate, batch_size, decay_steps, decay_rate, sequence_length, vocab_size, embed_size, is_training) saver = tf.train.Saver() if os.path.exists(ckpt_dir + 'checkpoint'): print('Restoring Variables from Checkpoint for rnn model.') saver.restore( sess, tf.train.latest_checkpoint(ckpt_dir)) # todo 怎么找到最近保存的文件的? else: print('Initializing Variables') sess.run(tf.global_variables_initializer()) if use_embedding: # load pre_trained word embedding assign_pretrained_word_embedding( sess, vocabulary_index2word, vocab_size, textRNN, word2vec_model_path=word2vec_model_path) curr_epoch = sess.run(textRNN.epoch_step) # 3.feed data & training number_of_training_data = len(trainX) for epoch in range(curr_epoch, num_epochs): loss, acc, counter = 0.0, 0.0, 0 for start, end in zip( range(0, number_of_training_data, batch_size), range(batch_size, number_of_training_data, batch_size)): if epoch == 0 and counter == 0: print('trainX[start:end]:', trainX[start:end]) curr_loss, curr_acc, _ = sess.run( [textRNN.loss_val, textRNN.accuracy, textRNN.train_op], feed_dict={ textRNN.input_x: trainX[start:end], textRNN.input_y: trainY[start:end], textRNN.dropout_keep_prob: 1.0 }) loss, counter, acc = loss + curr_loss, counter + 1, acc + curr_acc if counter % 500 == 0: print( 'Epoch {} \tBatch {}\tTrain Loss:{:.3}\tTrain Accuracy:{:.3}' .format(epoch, counter, loss / float(counter), acc / float(counter))) # epoch increament print('going to increament epoch counter ...') sess.run(textRNN.epoch_increament) # 4.validation print(epoch, validation_every, (epoch % validation_every == 0)) if epoch % validation_every == 0: eval_loss, eval_acc = do_eval(sess, textRNN, testX, testY, batch_size, vocabulary_index2word_label) print( 'Epoch {} Validation Loss: {:.3} \tValidation Accuracy:{:.3}' .format(epoch, eval_loss, eval_acc)) # save model to checkpoint save_path = ckpt_dir + 'model.ckpt' # saver.save(sess,save_path,global_step=epoch) saver.save(sess, save_path, global_step=textRNN.global_step) # 5. test in testData and report accuracy test_loss, test_acc = do_eval(sess, textRNN, testX, testY, batch_size, vocabulary_index2word_label) pass