def train(train_path='', test_path='', save_vocab_path='', attn_model_path='', batch_size=64, epochs=100, maxlen=400, hidden_dim=128, use_gpu=False): data_reader = CGEDReader(train_path) input_texts, target_texts = data_reader.build_dataset(train_path) test_input_texts, test_target_texts = data_reader.build_dataset(test_path) # load or save word dict if os.path.exists(save_vocab_path): char2id = load_word_dict(save_vocab_path) id2char = {int(j): i for i, j in char2id.items()} chars = set([i for i in char2id.keys()]) else: print('Training data...') print('input_texts:', input_texts[0]) print('target_texts:', target_texts[0]) max_input_texts_len = max([len(text) for text in input_texts]) print('num of samples:', len(input_texts)) print('max sequence length for inputs:', max_input_texts_len) chars = data_reader.read_vocab(input_texts + target_texts) id2char = {i: j for i, j in enumerate(chars)} char2id = {j: i for i, j in id2char.items()} save_word_dict(char2id, save_vocab_path) model = Seq2seqAttnModel(chars, attn_model_path=attn_model_path, hidden_dim=hidden_dim, use_gpu=use_gpu).build_model() evaluator = Evaluate(model, attn_model_path, char2id, id2char, maxlen) model.fit_generator(data_generator(input_texts, target_texts, char2id, batch_size, maxlen), steps_per_epoch=(len(input_texts) + batch_size - 1) // batch_size, epochs=epochs, validation_data=get_validation_data(test_input_texts, test_target_texts, char2id, maxlen), callbacks=[evaluator])
def __init__(self, save_vocab_path='', save_pinyin_path='', attn_model_path='', maxlen=400): if os.path.exists(save_vocab_path): self.char2id = load_word_dict(save_vocab_path) self.pinyins = load_word_dict(save_pinyin_path) self.id2char = {int(j): i for i, j in self.char2id.items()} self.id2pinyin = {int(j): i for i, j in self.pinyins.items()} self.chars = set([i for i in self.char2id.keys()]) self.pinyin2id = set([i for i in self.pinyins.keys()]) else: print('not exist vocab path') seq2seq_attn_model = Seq2seqAttn_multiembedding( self.chars, self.pinyin2id, attn_model_path=attn_model_path) self.model = seq2seq_attn_model.build_model() self.maxlen = maxlen
def __init__(self, save_vocab_path='', attn_model_path='', maxlen=400): if os.path.exists(save_vocab_path): self.char2id = load_word_dict(save_vocab_path) self.id2char = {int(j): i for i, j in self.char2id.items()} self.chars = set([i for i in self.char2id.keys()]) else: print('not exist vocab path') seq2seq_attn_model = Seq2seqAttnModel(self.chars, attn_model_path=attn_model_path, hidden_dim=config.rnn_hidden_dim, use_gpu=config.use_gpu) self.model = seq2seq_attn_model.build_model() self.maxlen = maxlen
import numpy as np from pycorrector.seq2seq_attention import config from pycorrector.seq2seq_attention.corpus_reader import CGEDReader, str2id, padding, load_word_dict, save_word_dict def data_generator(input_texts, target_texts, char2id, batch_size, maxlen=400): # 数据生成器 while True: X, Y = [], [] for i in range(len(input_texts)): X.append(str2id(input_texts[i], char2id, maxlen)) # print("X", X) Y.append(str2id(target_texts[i], char2id, maxlen)) # print("Y", Y) if len(X) == batch_size: X = np.array(padding(X, char2id)) Y = np.array(padding(Y, char2id)) # yield [X, Y], None X, Y = [], [] if __name__ == '__main__': train_path = config.test_path data_reader = CGEDReader(train_path) input_texts, target_texts = data_reader.build_dataset( train_path) # 原始文本,修改后文本 print("input_text:", input_texts) save_vocab_path = config.save_vocab_path char2id = load_word_dict(save_vocab_path) batch_size = config.batch_size data_generator(input_texts, target_texts, char2id, batch_size)
def train(train_path='', test_path='', save_vocab_path='', save_pinyin_path='', attn_model_path='', batch_size=64, epochs=100, maxlen=400, hidden_dim=128, dropout=0.2, use_gpu=False): data_reader = CGEDReader(train_path) input_texts, target_texts = data_reader.build_dataset( train_path) # 原始文本,修改后文本 # todo 获取相应拼音文本 input_pinyins = to_pinyin(input_texts) output_pinyins = to_pinyin(target_texts) test_input_texts, test_target_texts = data_reader.build_dataset(test_path) test_input_pinyins = to_pinyin(test_input_texts) test_output_pinyins = to_pinyin(test_target_texts) # load or save word dict if os.path.exists(save_vocab_path): char2id = load_word_dict(save_vocab_path) id2char = {int(j): i for i, j in char2id.items() } # {0: '"', 1: '%', 2: '(', 3: ')', 4: '+'} chars = set([i for i in char2id.keys()]) # {'+', '%', ')', '(', '"'} else: print('Training data...') print('input_texts:', input_texts[0]) print('target_texts:', target_texts[0]) max_input_texts_len = max([len(text) for text in input_texts]) print('num of samples:', len(input_texts)) print('max sequence length for inputs:', max_input_texts_len) # chars:保存所有字符,存放到列表中 chars = data_reader.read_vocab(input_texts + target_texts) id2char = {i: j for i, j in enumerate(chars)} char2id = {j: i for i, j in id2char.items()} save_word_dict(char2id, save_vocab_path) print("chars:", chars) # load pinyin vocab dict if os.path.exists(save_pinyin_path): pingyin2id = load_word_dict(save_pinyin_path) id2pinyin = {int(j): i for i, j in pingyin2id.items()} pinyins = set([i for i in pingyin2id.keys()]) else: pinyins = data_reader.read_vocab(input_pinyins + output_pinyins) id2pinyin = {i: j for i, j in enumerate(pinyins)} pingyin2id = {j: i for i, j in id2pinyin.items()} save_word_dict(pingyin2id, save_pinyin_path) print("pinyins:", pinyins) model = Seq2seqAttn_multiembedding(chars, pinyins, attn_model_path=attn_model_path, hidden_dim=hidden_dim, use_gpu=use_gpu, dropout=dropout).build_model() # evaluator = Evaluate(model, attn_model_path, char2id, id2char, maxlen) # model.fit_generator(data_generator(input_texts, target_texts, char2id, input_pinyins, output_pinyins, pingyin2id,batch_size, maxlen), # steps_per_epoch=(len(input_texts) + batch_size - 1) // batch_size, # epochs=epochs, # validation_data=get_validation_data(test_input_texts, test_target_texts, char2id,test_input_pinyins, test_output_pinyins, pingyin2id, maxlen), # callbacks=[evaluator]) model.fit_generator( data_generator(input_texts, target_texts, char2id, input_pinyins, output_pinyins, pingyin2id, batch_size, maxlen), steps_per_epoch=(len(input_texts) + batch_size - 1) // batch_size, epochs=epochs, validation_data=get_validation_data(test_input_texts, test_target_texts, char2id, test_input_pinyins, test_output_pinyins, pingyin2id, maxlen))