def get_cnn_result(): if not os.path.exists(embedding_model_path): print("word2vec model is not found") if not os.path.exists(train_data_path): print("train params is not found") params = readdata.loadDict(train_data_path) train_length = int(params['max_sentences_length']) test_sample_lists = readdata.get_cleaned_list(test_file_path) test_sample_lists, max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=train_length) test_sample_arrays = np.array( word2vec.get_embedding_vector(test_sample_lists, embedding_model_path)) testconfig = config() testconfig.max_sentences_length = max_sentences_length sess = tf.InteractiveSession() cnn = cnn_model.TextCNN(config=testconfig) #加载参数 saver = tf.train.Saver() saver.restore(sess, "data/cnn/text_model") #定义测试函数 def test_step(x_batch): feed_dict = {cnn.input_x: x_batch, cnn.dropout_keep_prob: 1.0} predictions, scores = sess.run([cnn.predictions, cnn.softmax_result], feed_dict=feed_dict) return (predictions, scores) #拿到结果 predictions, scores = test_step(test_sample_arrays) return np.array(predictions)
def get_mixed_result(): if not os.path.exists(embedding_model_path): print("word2vec model is not found") if not os.path.exists(lstm_train_data_path): print("lstm train params is not found") lstm_params = readdata.loadDict(lstm_train_data_path) lstm_train_length = int(lstm_params['max_sentences_length']) if not os.path.exists(cnn_train_data_path): print("cnn train params is not found") cnn_params = readdata.loadDict(cnn_train_data_path) cnn_train_length = int(cnn_params['max_sentences_length']) test_sample_lists = readdata.get_cleaned_list(test_file_path) lstm_test_sample_lists, lstm_max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=lstm_train_length) cnn_test_sample_lists, cnn_max_sentences_length = readdata.padding_sentences( test_sample_lists, padding_token='<PADDING>', padding_sentence_length=cnn_train_length) lstm_test_sample_arrays = np.array( word2vec.get_embedding_vector(lstm_test_sample_lists, embedding_model_path)) cnn_test_sample_arrays = np.array( word2vec.get_embedding_vector(cnn_test_sample_lists, embedding_model_path)) lstm_config = lstmconfig() cnn_config = cnnconfig() lstm_config.max_sentences_length = lstm_max_sentences_length cnn_config.max_sentences_length = cnn_max_sentences_length lstm_graph = tf.Graph() cnn_graph = tf.Graph() lstm_sess = tf.Session(graph=lstm_graph) cnn_sess = tf.Session(graph=cnn_graph) with lstm_sess.as_default(): with lstm_graph.as_default(): lstm = lstm_model.TextLSTM(config=lstm_config) lstm_saver = tf.train.Saver() lstm_saver.restore(lstm_sess, "data/lstm/text_model") def lstm_test_step(x_batch): feed_dict = { lstm.input_x: x_batch, lstm.dropout_keep_prob: lstm_config.dropout_keep_prob } scores = lstm_sess.run([lstm.softmax_result], feed_dict=feed_dict) return scores lstm_scores = lstm_test_step(lstm_test_sample_arrays) with cnn_sess.as_default(): with cnn_graph.as_default(): cnn = cnn_model.TextCNN(config=cnn_config) cnn_saver = tf.train.Saver() cnn_saver.restore(cnn_sess, "data/cnn/text_model") def cnn_test_step(x_batch): feed_dict = { cnn.input_x: x_batch, cnn.dropout_keep_prob: cnn_config.dropout_keep_prob } scores = cnn_sess.run([cnn.softmax_result], feed_dict=feed_dict) return scores cnn_scores = cnn_test_step(cnn_test_sample_arrays) lstm_sess.close() cnn_sess.close() mixed_scores = np.sum([lstm_scores, cnn_scores], axis=0) predictions = np.argmax(mixed_scores, axis=2) return np.array(predictions)
x_train = np.array(list(x_transform_train)) x_test = np.array(list(x_transform_test)) y_train = np.array(list(y_train)) y_test = np.array(list(y_test)) embedding_dim = 128 filter_sizes = '3,4,5' l2_reg_lambda = 0.0 num_filters = 128 with tf.Session() as sess: cnn = cnn_model.TextCNN(sequence_length=dataset.get_max_len(), num_classes=len(config.classes), vocab_size=len(vocab_processor.vocabulary_), embedding_size=embedding_dim, filter_sizes=list(map(int, filter_sizes.split(","))), num_filters=num_filters, l2_reg_lambda=l2_reg_lambda) out_dir = os.path.abspath(os.path.join('tmp', "runs/")) print("Writing to {}\n".format(out_dir)) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, cnn.grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, "summaries", "train")
num_tests = int(trainconfig.test_sample_percentage * len(all_label_arrays)) del all_label_arrays, random_index test_sample_arrays = random_sample_arrays[:num_tests] train_sample_arrays = random_sample_arrays[num_tests:] del random_sample_arrays test_label_arrays = random_label_arrays[:num_tests] train_label_arrays = random_label_arrays[num_tests:] del random_label_arrays print("Train/Test split: {:d}/{:d}".format(len(train_label_arrays), len(test_label_arrays))) #开始训练 with tf.Graph().as_default(): sess = tf.Session() with sess.as_default(): cnn = cnn_model.TextCNN(config=trainconfig) #初始化参数 train_writer = tf.summary.FileWriter(log_path + '/train', sess.graph) test_writer = tf.summary.FileWriter(log_path + '/test') step_num = 0 sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() #定义训练函数 def train_step(x_batch, y_batch, lr_rate): feed_dict = { cnn.input_x: x_batch, cnn.input_y: y_batch, cnn.dropout_keep_prob: trainconfig.dropout_keep_prob, cnn.learning_rate: lr_rate