def predict(): word_weights, tag_weights = load_embedding() word_voc, tag_voc, label_voc = load_voc() # train data sentences, tags, labels = load_train_data(word_voc, tag_voc, label_voc) seed = 137 np.random.seed(seed) np.random.shuffle(sentences) np.random.seed(seed) np.random.shuffle(tags) np.random.seed(seed) np.random.shuffle(labels) # load data sentences_test, tags_test = load_test_data(word_voc, tag_voc, label_voc) labels_test = None # clear reslut command = 'rm ./Data/result/*' os.popen(command) # 划分训练、开发、测试集 kf = KFold(n_splits=config.KFOLD) train_indices, dev_indices = [], [] for train_index, dev_index in kf.split(labels): train_indices.append(train_index) dev_indices.append(dev_index) for num in range(config.KFOLD): train_index, dev_index = train_indices[num], dev_indices[num] sentences_train, sentences_dev = sentences[train_index], sentences[dev_index] tags_train, tags_dev = tags[train_index], tags[dev_index] labels_train, labels_dev = labels[train_index], labels[dev_index] # init model model = DCModel( config.MAX_LEN, word_weights, tag_weights, result_path='./Data/result/result.txt', label_voc=label_voc) # fit model model.fit( sentences_train, tags_train, labels_train, sentences_dev, tags_dev, labels_dev, sentences_test, tags_test, labels_test, config.BATCH_SIZE, config.NB_EPOCH, keep_prob=config.KEEP_PROB, word_keep_prob=config.WORD_KEEP_PROB, tag_keep_prob=config.TAG_KEEP_PROB) print(model.get_best_score()) [p_test, r_test, f_test], nb_epoch = model.get_best_score() command = 'cp ./Data/result/epoch_%d.csv ./Data/result/best_%d' % (nb_epoch+1, num) print(command) os.popen(command) print(p_test, r_test, f_test, '\n') # evaluate # result_path_k = result_path % k # p_test, r_test, f_test = model.evaluate(sentences_test, tags_test, positions_test, # labels_test, simple_compute=False, ignore_label=IGNORE_LABEL, # label_voc=relation_voc, result_path=result_path_k) # clear model model.clear_model() del model
def main_lstmatt(args, DEVICE): train_iter, dev_iter, test_iter, text_field, label_field = load_data(args, DEVICE) pretrained_embeddings = load_embedding(text_field) model = LSTMAttBC(args.dim_embd, args.dim, args.batch_size, pretrained_embeddings, vocab_size=len(text_field.vocab), label_size=len(label_field.vocab) - 1, DEVICE=DEVICE)
def main_lstmatt(args, DEVICE): train_iter, dev_iter, test_iter, text_field, label_field = load_data( args, DEVICE) pretrained_embeddings = load_embedding(text_field) model = LSTMAttBC(args.dim_embd, args.dim, args.batch_size, pretrained_embeddings, vocab_size=len(text_field.vocab), label_size=len(label_field.vocab) - 1, DEVICE=DEVICE) # Train if no pre-trained model is given if args.new_train: train(args, [train_iter, dev_iter, test_iter], model, DEVICE) interpret_lstmatt(args, test_iter, text_field, label_field, DEVICE)
#predictions: [candidate_answer_num, 2* hidden_dim] with tf.variable_scope('projection_layer',reuse=True): softmax_w=tf.get_variable('softmax_w') softmax_b=tf.get_variable('softmax_b') scores=tf.matmul(predictions, softmax_w)+softmax_b scores=tf.squeeze(scores) #[candidate_answer_num] scores=tf.expand_dims(scores, 0) #[1, candidate_answer_num] truth=tf.onehot(answer, tf.gather(tf.shape(predictions),0)) truth=tf.expand_dims(truth, 0) cross_entropy=tf.nn.softmax_cross_entropy_with_logits(logits=scores, labels=truth) cross_entropy=tf.squeeze(cross_entropy) return cross_entropy def add_training_op(self): opt=tr.train.AdagradOptimizer(self.config.lr) train_op=opt.minimize(self.loss) return train_op if __name__=='__main__': logging.basicConfig(filename="ccrc_model_test.log",level=logging.WARNING) from my_main import Config config=Config() word2idx,embedding=load_data.load_embedding() config.embedding=embedding gpu_config = tf.ConfigProto() gpu_config.gpu_options.allow_growth = True with tf.Session(config=gpu_config) as sess: logging.warn('begin build the model') model = ccrc_model(config) logging.warn('model build done') sess.run(tf.global_variables_initializer())
# -*- coding: utf-8 -*- from load_data import load_embedding, load_voc, load_train_data, load_test_data import time from generator import BatchGenerator from TFNN.layers.EmbeddingLayer import Embedding from sklearn.model_selection import KFold from triggerType_to_trigger import get_trigger ''' For Chinese word segmentation. ''' #############################1.load data ###################################### class_type = 3 training_count = 16796 test_count = 2570 word_weights, tag_weights = load_embedding() #矩阵形式 word_voc, tag_voc, label_voc = load_voc() #字典形式 sentences, tags, labels = load_train_data(word_voc, tag_voc, label_voc, class_type, training_count) Xend_sentence, Xend_tag_test, yend_test = load_test_data( word_voc, tag_voc, label_voc, class_type, test_count) #划分训练集,测试集(这里的y为词性tag kf = KFold(n_splits=10) train_indices, dev_indices = [], [] for train_index, dev_index in kf.split(labels): train_indices.append(train_index) dev_indices.append(dev_index) for num in range(10): train_index, dev_index = train_indices[num], dev_indices[num]