def get_lstm_result(): if not os.path.exists(embedding_model_path): print("word2vec model is not found") if not os.path.exists(train_data_path): print("train params is not found") params = readdata.loadDict(train_data_path) train_length = int(params['max_sentences_length']) test_sample_lists = readdata.get_cleaned_list(test_file_path) test_sample_lists,max_sentences_length = readdata.padding_sentences(test_sample_lists,padding_token='<PADDING>',padding_sentence_length=train_length) test_sample_arrays=np.array(word2vec.get_embedding_vector(test_sample_lists,embedding_model_path)) testconfig=config() testconfig.max_sentences_length=max_sentences_length sess=tf.InteractiveSession() lstm=lstm_model.TextLSTM(config=testconfig) saver = tf.train.Saver() saver.restore(sess, "data/lstm/text_model") #定义测试函数 def test_step(x_batch): feed_dict={ lstm.input_x:x_batch, lstm.dropout_keep_prob:testconfig.dropout_keep_prob } predictions,scores=sess.run( [lstm.predictions,lstm.softmax_result], feed_dict=feed_dict ) return (predictions,scores) predictions, scores=test_step(test_sample_arrays) return np.array(predictions)
def get_mixed_result(): if not os.path.exists(embedding_model_path): print("word2vec model is not found") if not os.path.exists(lstm_train_data_path): print("lstm train params is not found") lstm_params = readdata.loadDict(lstm_train_data_path) lstm_train_length = int(lstm_params['max_sentences_length']) if not os.path.exists(cnn_train_data_path): print("cnn train params is not found") cnn_params = readdata.loadDict(cnn_train_data_path) cnn_train_length = int(cnn_params['max_sentences_length']) test_sample_lists = readdata.get_cleaned_list(test_file_path) lstm_test_sample_lists, lstm_max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=lstm_train_length) cnn_test_sample_lists, cnn_max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=cnn_train_length) lstm_test_sample_arrays = np.array( word2vec.get_embedding_vector(lstm_test_sample_lists, embedding_model_path)) cnn_test_sample_arrays = np.array( word2vec.get_embedding_vector(cnn_test_sample_lists, embedding_model_path)) lstm_config = lstmconfig() cnn_config = cnnconfig() lstm_config.max_sentences_length = lstm_max_sentences_length cnn_config.max_sentences_length = cnn_max_sentences_length lstm_graph = tf.Graph() cnn_graph = tf.Graph() lstm_sess = tf.Session(graph=lstm_graph) cnn_sess = tf.Session(graph=cnn_graph) with lstm_sess.as_default(): with lstm_graph.as_default(): lstm = lstm_model.TextLSTM(config=lstm_config) lstm_saver = tf.train.Saver() lstm_saver.restore(lstm_sess, "data/lstm/text_model") def lstm_test_step(x_batch): feed_dict = { lstm.input_x: x_batch, lstm.dropout_keep_prob: lstm_config.dropout_keep_prob } scores = lstm_sess.run([lstm.softmax_result], feed_dict=feed_dict) return scores lstm_scores = lstm_test_step(lstm_test_sample_arrays) with cnn_sess.as_default(): with cnn_graph.as_default(): cnn = cnn_model.TextCNN(config=cnn_config) cnn_saver = tf.train.Saver() cnn_saver.restore(cnn_sess, "data/cnn/text_model") def cnn_test_step(x_batch): feed_dict = { cnn.input_x: x_batch, cnn.dropout_keep_prob: cnn_config.dropout_keep_prob } scores = cnn_sess.run([cnn.softmax_result], feed_dict=feed_dict) return scores cnn_scores = cnn_test_step(cnn_test_sample_arrays) lstm_sess.close() cnn_sess.close() mixed_scores = np.sum([lstm_scores, cnn_scores], axis=0) predictions = np.argmax(mixed_scores, axis=2) return np.array(predictions)
test_sample_percentage = 0.1 #训练样本和测试样本的比例 num_labels = 2 #标签数量 embedding_size = 64 #词向量维度 dropout_keep_prob = 1 #dropout数量 batch_size = 64 num_epochs = 80 max_sentences_length = 100 num_layers = 2 max_grad_norm = 5 l2_rate = 0.000005 params = readdata.loadDict(train_data_path) testconfig = config() lstm = lstm_model.TextLSTM(config=testconfig) sess = tf.InteractiveSession() saver = tf.train.Saver() saver.restore(sess, "./data/lstm/text_model") def get_lstm_result(test_sample): train_length = int(params['max_sentences_length']) test_sample_lists = readdata.get_cleaned_list1(test_sample) test_sample_lists, max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=train_length) test_sample_arrays = np.array(get_embedding_vector(test_sample_lists)) def test_step(x_batch):
num_tests = int(trainconfig.test_sample_percentage * len(all_label_arrays)) del all_label_arrays test_sample_arrays = random_sample_arrays[:num_tests] train_sample_arrays = random_sample_arrays[num_tests:] del random_sample_arrays train_label_arrays = random_label_arrays[num_tests:] test_label_arrays = random_label_arrays[:num_tests] del random_label_arrays print("Train/Test split: {:d}/{:d}".format(len(train_label_arrays), len(test_label_arrays))) #开始训练 with tf.Graph().as_default(): sess = tf.Session() with sess.as_default(): lstm = lstm_model.TextLSTM(config=trainconfig) #初始化参数 train_writer = tf.summary.FileWriter(log_path + '/train', sess.graph) test_writer = tf.summary.FileWriter(log_path + '/test') step_num = 0 sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() #定义训练函数 def train_step(x_batch, y_batch): feed_dict = { lstm.input_x: x_batch, lstm.input_y: y_batch, lstm.dropout_keep_prob: config.dropout_keep_prob, }