def train_model(args): trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(args.source), args.vocab) trg_vocab = Vocabulary.new(gens.word_list(args.target), args.vocab) trace('making model ...') model = EncoderDecoderModel.new(src_vocab, trg_vocab, args.embed, args.hidden) for epoch in range(args.epoch): trace('epoch %d/%d: ' % (epoch + 1, args.epoch)) trained = 0 gen1 = gens.word_list(args.source) gen2 = gens.word_list(args.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * args.minibatch), args.minibatch) model.init_optimizer() for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch = model.train(src_batch, trg_batch) for k in range(K): trace('epoch %3d/%3d, sample %8d' % (epoch + 1, args.epoch, trained + k + 1)) trace(' src = ' + ' '.join([x if x != '</s>' else '*' for x in src_batch[k]])) trace(' trg = ' + ' '.join([x if x != '</s>' else '*' for x in trg_batch[k]])) trace(' hyp = ' + ' '.join([x if x != '</s>' else '*' for x in hyp_batch[k]])) trained += K trace('saving model ...') model.save(args.model + '.%03d' % (epoch + 1)) trace('finished.')
def train_model(args): trace('making vocaburaries ...') src_vocab = Vocabulary.new(gens.word_list(args.source), args.vocab) trg_vocab = Vocabulary.new(gens.word_list(args.target), args.vocab) trace('making model ...') model = EncoderDecoderModel.new(src_vocab, trg_vocab, args.embed, args.hidden) for epoch in range(args.epoch): trace('epoch %d/%d: ' % (epoch + 1, args.epoch)) trained = 0 gen1 = gens.word_list(args.source) gen2 = gens.word_list(args.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * args.minibatch), args.minibatch) model.init_optimizer() for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch = model.train(src_batch, trg_batch) for k in range(K): trace('epoch %3d/%3d, sample %8d' % (epoch + 1, args.epoch, trained + k + 1)) trace(' src = ' + ' '.join([x if x != '</s>' else '*' for x in src_batch[k]])) trace(' trg = ' + ' '.join([x if x != '</s>' else '*' for x in trg_batch[k]])) trace(' hyp = ' + ' '.join([x if x != '</s>' else '*' for x in hyp_batch[k]])) trained += K trace('saving model ...') model.save(args.model + '.%03d' % (epoch + 1)) trace('finished.')
def train_model(self): trace('making vocaburaries ...') src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace('making model ...') model = self.new(src_vocab, trg_vocab, self.embed, self.hidden, self.parameter_dict) random_number = random.randint(0, self.minibatch) for i_epoch in range(self.epoch): trace('epoch %d/%d: ' % (i_epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) model.init_optimizer() for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch = model.train(src_batch, trg_batch) if trained == 0: self.print_out(random_number, i_epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace('saving model ...') model.save("ChainerMachineTranslation" + '.%03d' % (self.epoch + 1)) trace('finished.')
def train(args): trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(args.source), args.vocab) trg_vocab = Vocabulary.new(gens.word_list(args.target), args.vocab) trace('making model ...') attmt = AttentionMT(args.vocab, args.embed, args.hidden) if args.use_gpu: attmt.to_gpu() for epoch in range(args.epoch): trace('epoch %d/%d: ' % (epoch + 1, args.epoch)) trained = 0 gen1 = gens.word_list(args.source) gen2 = gens.word_list(args.target) gen3 = gens.batch( gens.sorted_parallel(gen1, gen2, 100 * args.minibatch), args.minibatch) opt = optimizers.AdaGrad(lr=0.01) opt.setup(attmt) opt.add_hook(optimizer.GradientClipping(5)) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch, loss = forward(src_batch, trg_batch, src_vocab, trg_vocab, attmt, True, 0) loss.backward() opt.update() for k in range(K): trace('epoch %3d/%3d, sample %8d' % (epoch + 1, args.epoch, trained + k + 1)) trace( ' src = ' + ' '.join([x if x != '</s>' else '*' for x in src_batch[k]])) trace( ' trg = ' + ' '.join([x if x != '</s>' else '*' for x in trg_batch[k]])) trace( ' hyp = ' + ' '.join([x if x != '</s>' else '*' for x in hyp_batch[k]])) trained += K trace('saving model ...') prefix = args.model + '.%03.d' % (epoch + 1) src_vocab.save(prefix + '.srcvocab') trg_vocab.save(prefix + '.trgvocab') attmt.save_spec(prefix + '.spec') serializers.save_hdf5(prefix + '.weights', attmt) trace('finished.')
def train(self): """ Train method If you use the word2vec model, you possible to use the copy weight Optimizer method use the Adagrad """ trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace('making model ...') self.attention_dialogue = AttentionDialogue(self.vocab, self.embed, self.hidden, self.XP) if self.word2vecFlag: self.copy_model(self.word2vec, self.attention_dialogue.emb) self.copy_model(self.word2vec, self.attention_dialogue.dec, dec_flag=True) for epoch in range(self.epoch): trace('epoch %d/%d: ' % (epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch( gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) opt = optimizers.AdaGrad(lr=0.01) opt.setup(self.attention_dialogue) opt.add_hook(optimizer.GradientClipping(5)) random_number = random.randint(0, self.minibatch - 1) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch, loss = self.forward_implement( src_batch, trg_batch, src_vocab, trg_vocab, self.attention_dialogue, True, 0) loss.backward() opt.update() self.print_out(random_number, epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace('saving model ...') prefix = self.model model_path = APP_ROOT + "/model/" + prefix src_vocab.save(model_path + '.srcvocab') trg_vocab.save(model_path + '.trgvocab') self.attention_dialogue.save_spec(model_path + '.spec') serializers.save_hdf5(model_path + '.weights', self.attention_dialogue) trace('finished.')
def train(args): if os.path.exists("./model/vocab.bin"): src_vocab = Vocabulary.load("./model/vocab.bin") else: src_vocab = Vocabulary.new(gens.word_list(args.source), args.n_vocab) src_vocab.save('./model/vocab.bin') if os.path.exists("./model/tag.bin"): trg_tag = Vocabulary.load("./model/tag.bin") else: trg_tag = Vocabulary.new(gens.word_list(args.target), args.n_tag) trg_tag.save('./model/tag.bin') print("vocab_len:{}".format(src_vocab.__len__)) print("tag_len:{}".format(trg_tag.__len__)) encdec = BiEncDecLSTM(args.n_vocab, args.layer, args.embed, args.hidden, args.n_tag) optimizer = optimizers.Adam() optimizer.setup(encdec) for e_i in range(args.epoch): tt_list = [[src_vocab.stoi(char) for char in char_arr] for char_arr in gens.word_list(args.source_tr)] tag_list = [ trg_tag.stoi(tag[0]) for tag in gens.word_list(args.target_tr) ] print("{}:{}".format(len(tt_list), len(tag_list))) assert len(tt_list) == len(tag_list) ind_arr = [ri for ri in range(len(tt_list))] random.shuffle(ind_arr) tt_now = (tt_list[ri] for ri in ind_arr) tag_now = (tag_list[ri] for ri in ind_arr) tt_gen = gens.batch(tt_now, args.batchsize) tag_gen = gens.batch(tag_now, args.batchsize) for tt, tag in zip(tt_gen, tag_gen): y_ws = encdec(tt) teac_arr = [src_vocab.itos(t) for t in tt[0]] pred_arr = [trg_tag.itos(y_each.data.argmax(0)) for y_each in y_ws] print("teach:{}:{}:{}".format(teac_arr, trg_tag.itos(tag[0]), pred_arr[0])) tag = xp.array(tag, dtype=xp.int32) loss = F.softmax_cross_entropy(y_ws, tag) encdec.cleargrads() loss.backward() optimizer.update() # loss.backward() # optimizer.target.cleargrads() # loss.backward() # loss.unchain_backward() # optimizer.update() serializers.save_npz('./model/attn_tag_model_{}.npz'.format(e_i), encdec)
def setCateg(self, args): categ_name = "./{}/categ_{}.bin".format(args.dataname, args.dataname) if os.path.exists(categ_name): categ_vocab = Vocabulary.load(categ_name) else: set_cat = set() [[set_cat.add(word) for word in word_arr] for word_arr in gens.word_list(args.category)] n_categ = len(set_cat) + 3 print("n_categ:{}".format(n_categ)) categ_vocab = Vocabulary.new(gens.word_list(args.category), n_categ) categ_vocab.save(categ_name) self.categ_vocab = categ_vocab return categ_vocab
def getBatchGen(self, args): tt_now_list = [[self.vocab.stoi(char) for char in char_arr] for char_arr in gens.word_list(args.source)] cat_now_list = [[self.categ_vocab.stoi(cat[0])] for cat in gens.word_list(args.category)] ind_arr = list(range(len(tt_now_list))) random.shuffle(ind_arr) tt_now = (tt_now_list[ind] for ind in ind_arr) cat_now = (cat_now_list[ind] for ind in ind_arr) tt_gen = gens.batch(tt_now, args.batchsize) cat_gen = gens.batch(cat_now, args.batchsize) for tt, cat in zip(tt_gen, cat_gen): yield (tt, cat)
def setVocab(self, args): vocab_name = "./{}/vocab_{}.bin".format(args.dataname, args.dataname) if os.path.exists(vocab_name): src_vocab = Vocabulary.load(vocab_name) else: set_vocab = set() [[set_vocab.add(word) for word in word_arr] for word_arr in gens.word_list(args.source)] n_vocab = len(set_vocab) + 3 print("n_vocab:{}".format(n_vocab)) print("arg_vocab:{}".format(args.n_vocab)) src_vocab = Vocabulary.new(gens.word_list(args.source), args.n_vocab) src_vocab.save(vocab_name) self.vocab = src_vocab return src_vocab
def train(self): """ Train method If you use the word2vec model, you possible to use the copy weight Optimizer method use the Adagrad """ trace("making vocabularies ...") src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace("making model ...") self.attention_dialogue = AttentionDialogue(self.vocab, self.embed, self.hidden, self.XP) if self.word2vecFlag: self.copy_model(self.word2vec, self.attention_dialogue.emb) self.copy_model(self.word2vec, self.attention_dialogue.dec, dec_flag=True) for epoch in range(self.epoch): trace("epoch %d/%d: " % (epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) opt = optimizers.AdaGrad(lr=0.01) opt.setup(self.attention_dialogue) opt.add_hook(optimizer.GradientClipping(5)) random_number = random.randint(0, self.minibatch - 1) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch, loss = self.forward_implement( src_batch, trg_batch, src_vocab, trg_vocab, self.attention_dialogue, True, 0 ) loss.backward() opt.update() self.print_out(random_number, epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace("saving model ...") prefix = self.model model_path = APP_ROOT + "/model/" + prefix src_vocab.save(model_path + ".srcvocab") trg_vocab.save(model_path + ".trgvocab") self.attention_dialogue.save_spec(model_path + ".spec") serializers.save_hdf5(model_path + ".weights", self.attention_dialogue) trace("finished.")
def train(self): trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace('making model ...') encdec = EncoderDecoder(self.vocab, self.embed, self.hidden) if self.word2vecFlag: self.copy_model(self.word2vec, encdec.enc) self.copy_model(self.word2vec, encdec.dec, dec_flag=True) for epoch in range(self.epoch): trace('epoch %d/%d: ' % (epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch( gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) opt = optimizers.AdaGrad(lr=0.01) opt.setup(encdec) opt.add_hook(optimizer.GradientClipping(5)) random_number = random.randint(0, self.minibatch - 1) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) # If you use the ipython note book you hace to use the forward function # hyp_batch, loss = self.forward(src_batch, trg_batch, src_vocab, trg_vocab, encdec, True, 0) hyp_batch, loss = self.forward_implement( src_batch, trg_batch, src_vocab, trg_vocab, encdec, True, 0) loss.backward() opt.update() self.print_out(random_number, epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace('saving model ...') prefix = self.model src_vocab.save(prefix + '.srcvocab') trg_vocab.save(prefix + '.trgvocab') encdec.save_spec(prefix + '.spec') serializers.save_hdf5(prefix + '.weights', encdec) trace('finished.')
def test(self): trace('loading model ...') src_vocab = Vocabulary.load(self.model + '.srcvocab') trg_vocab = Vocabulary.load(self.model + '.trgvocab') encdec = EncoderDecoder.load_spec(self.model + '.spec') serializers.load_hdf5(self.model + '.weights', encdec) trace('generating translation ...') generated = 0 with open(self.target, 'w') as fp: for src_batch in gens.batch(gens.word_list(self.source), self.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = self.forward(src_batch, None, src_vocab, trg_vocab, encdec, False, self.generation_limit) source_cuont = 0 for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] print("src : " + "".join(src_batch[source_cuont]).replace("</s>", "")) print('hyp : ' + ''.join(hyp)) print(' '.join(hyp), file=fp) source_cuont = source_cuont + 1 generated += K trace('finished.')
def __init__(self): self.source = "../data/serif8000_fixed.txt" self.target = "../data/tag.txt" self.source_tr = "../data/serif_tr8000_fixed.txt" self.target_tr = "../data/tag_tr.txt" # self.source_te ="../data/tweet.txt" self.source_te = "../data/serif_te8000_fixed.txt" self.target_te = "../data/tag_te.txt" self.epoch = 5 word_set = set() tag_set = set() [[word_set.add(word) for word in word_arr] for word_arr in gens.word_list(self.source)] [ tag_set.add(tag) for tag in codecs.open(self.target, "r", encoding="utf-8") ] self.n_vocab = len(word_set) self.n_tag = len(tag_set) self.embed = 50 self.hidden = 100 self.layer = 1 self.batchsize = 30 self.dropout = 0.5 self.gpu = -1 if self.gpu >= 0: import cupy as xp self.gradclip = 5
def test_model(self, model_name): trace('loading model ...') model = self.load(model_name) trace('generating translation ...') generated = 0 with open(self.test_target, 'w') as fp: for src_batch in gens.batch(gens.word_list(self.test_source), self.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = model.predict(src_batch, self.generation_limit) source_cuont = 0 for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] # BLEUの結果を計算するため. print("".join(src_batch[source_cuont]).replace("</s>", "")) print(' '.join(hyp)) print(' '.join(hyp), file=fp) source_cuont = source_cuont + 1 generated += K trace('finished.')
def test(self): trace('loading model ...') src_vocab = Vocabulary.load(self.model + '.srcvocab') trg_vocab = Vocabulary.load(self.model + '.trgvocab') encdec = EncoderDecoder.load_spec(self.model + '.spec') serializers.load_hdf5(self.model + '.weights', encdec) trace('generating translation ...') generated = 0 with open(self.target, 'w') as fp: for src_batch in gens.batch(gens.word_list(self.source), self.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = self.forward(src_batch, None, src_vocab, trg_vocab, encdec, False, self.generation_limit) source_cuont = 0 for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] print("src : " + "".join(src_batch[source_cuont]).replace("</s>", "")) print('hyp : ' +''.join(hyp)) print(' '.join(hyp), file=fp) source_cuont = source_cuont + 1 generated += K trace('finished.')
def test(args): trace('loading model ...') src_vocab = Vocabulary.load(args.model + '.srcvocab') trg_vocab = Vocabulary.load(args.model + '.trgvocab') attmt = AttentionMT.load_spec(args.model + '.spec') if args.use_gpu: attmt.to_gpu() serializers.load_hdf5(args.model + '.weights', attmt) trace('generating translation ...') generated = 0 with open(args.target, 'w') as fp: for src_batch in gens.batch(gens.word_list(args.source), args.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = forward(src_batch, None, src_vocab, trg_vocab, attmt, False, args.generation_limit) for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] print(' '.join(hyp), file=fp) generated += K trace('finished.')
def train(self): trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace('making model ...') encdec = EncoderDecoder(self.vocab, self.embed, self.hidden) if self.word2vecFlag: self.copy_model(self.word2vec, encdec.enc) self.copy_model(self.word2vec, encdec.dec, dec_flag=True) else: encdec = self.encdec for epoch in range(self.epoch): trace('epoch %d/%d: ' % (epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) opt = optimizers.AdaGrad(lr = 0.01) opt.setup(encdec) opt.add_hook(optimizer.GradientClipping(5)) random_number = random.randint(0, self.minibatch - 1) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch, loss = self.forward(src_batch, trg_batch, src_vocab, trg_vocab, encdec, True, 0) loss.backward() opt.update() if trained == 0: self.print_out(random_number, epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace('saving model ...') prefix = self.model src_vocab.save(prefix + '.srcvocab') trg_vocab.save(prefix + '.trgvocab') encdec.save_spec(prefix + '.spec') serializers.save_hdf5(prefix + '.weights', encdec) trace('finished.')
def getBatchGen(self, args): tt_now_list = [[self.vocab.stoi(char) for char in char_arr] for char_arr in gens.word_list(args.source)] ind_arr = list(range(len(tt_now_list))) random.shuffle(ind_arr) tt_now = (tt_now_list[ind] for ind in ind_arr) tt_gen = gens.batch(tt_now, args.batchsize) for tt in tt_gen: yield tt
def train(args): trace('making vocabularies ...') src_vocab = Vocabulary.new(gens.word_list(args.source), args.vocab) trg_vocab = Vocabulary.new(gens.word_list(args.target), args.vocab) trace('making model ...') attmt = AttentionMT(args.vocab, args.embed, args.hidden) if args.use_gpu: attmt.to_gpu() for epoch in range(args.epoch): trace('epoch %d/%d: ' % (epoch + 1, args.epoch)) trained = 0 gen1 = gens.word_list(args.source) gen2 = gens.word_list(args.target) gen3 = gens.batch(gens.sorted_parallel(gen1, gen2, 100 * args.minibatch), args.minibatch) opt = optimizers.AdaGrad(lr = 0.01) opt.setup(attmt) opt.add_hook(optimizer.GradientClipping(5)) for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch, loss = forward(src_batch, trg_batch, src_vocab, trg_vocab, attmt, True, 0) loss.backward() opt.update() for k in range(K): trace('epoch %3d/%3d, sample %8d' % (epoch + 1, args.epoch, trained + k + 1)) trace(' src = ' + ' '.join([x if x != '</s>' else '*' for x in src_batch[k]])) trace(' trg = ' + ' '.join([x if x != '</s>' else '*' for x in trg_batch[k]])) trace(' hyp = ' + ' '.join([x if x != '</s>' else '*' for x in hyp_batch[k]])) trained += K trace('saving model ...') prefix = args.model + '.%03.d' % (epoch + 1) src_vocab.save(prefix + '.srcvocab') trg_vocab.save(prefix + '.trgvocab') attmt.save_spec(prefix + '.spec') serializers.save_hdf5(prefix + '.weights', attmt) trace('finished.')
def test(args, epoch): model_name = "./model/attn_tag_model_{}.npz".format(epoch) encdec = BiEncDecLSTM(args.n_vocab, args.layer, args.embed, args.hidden, args.n_tag) serializers.load_npz(model_name, encdec) src_vocab = Vocabulary.load("./model/vocab.bin") trg_tag = Vocabulary.load("./model/tag.bin") tt_now = ([src_vocab.stoi(char) for char in char_arr] for char_arr in gens.word_list(args.source_te)) tag_now = (trg_tag.stoi(tag[0]) for tag in gens.word_list(args.target_te)) tt_gen = gens.batch(tt_now, args.batchsize) tag_gen = gens.batch(tag_now, args.batchsize) correct_num = 0 wrong_num = 0 fw = codecs.open("./output/result_attn_tw{}.csv".format(epoch), "w", encoding="utf-8") fw.write("台詞,教師キャラ,予測キャラ,予測値,›単語\n") for tt, tag in zip(tt_gen, tag_gen): y, att_w = encdec.callAndAtt(tt) max_y = [ max( F.softmax(F.reshape(y_each.data, (1, len(y_each.data)))).data[0]) for y_each in y ] y = [y_each.data.argmax(0) for y_each in y] for tt_e, y_e, tag_e, max_y_e, att_w_e in zip(tt, y, tag, max_y, att_w): txt = ",".join([src_vocab.itos(id) for id in tt_e]) tag_e = trg_tag.itos(tag_e) y_e = trg_tag.itos(y_e) att_ind = att_w_e.data.argmax() most_word = src_vocab.itos(tt_e[att_ind]) fw.write("{}:{}:{}:{}:{}\n".format(txt, tag_e, y_e, max_y_e, most_word)) correct_num += len([1 for y_e, tag_e in zip(y, tag) if y_e == tag_e]) wrong_num += len([1 for y_e, tag_e in zip(y, tag) if y_e != tag_e]) print("epoch:{}".format(epoch)) print(" correct:{}".format(correct_num)) print(" wrong:{}".format(wrong_num)) fw.write("correct{}\n".format(correct_num)) fw.write("wrong:{}\n".format(wrong_num)) fw.close()
def train_model(self): trace('making vocaburaries ...') src_vocab = Vocabulary.new(gens.word_list(self.source), self.vocab) trg_vocab = Vocabulary.new(gens.word_list(self.target), self.vocab) trace('making model ...') model = self.new(src_vocab, trg_vocab, self.embed, self.hidden, self.parameter_dict) random_number = random.randint(0, self.minibatch) for i_epoch in range(self.epoch): trace('epoch %d/%d: ' % (i_epoch + 1, self.epoch)) trained = 0 gen1 = gens.word_list(self.source) gen2 = gens.word_list(self.target) gen3 = gens.batch( gens.sorted_parallel(gen1, gen2, 100 * self.minibatch), self.minibatch) model.init_optimizer() for src_batch, trg_batch in gen3: src_batch = fill_batch(src_batch) trg_batch = fill_batch(trg_batch) K = len(src_batch) hyp_batch = model.train(src_batch, trg_batch) if trained == 0: self.print_out(random_number, i_epoch, trained, src_batch, trg_batch, hyp_batch) trained += K trace('saving model ...') model.save("ChainerMachineTranslation" + '.%03d' % (self.epoch + 1)) trace('finished.')
def test(self): """ Test method You have to parepare the train model """ trace('loading model ...') prefix = self.model model_path = APP_ROOT + "/model/" + prefix src_vocab = Vocabulary.load(model_path + '.srcvocab') trg_vocab = Vocabulary.load(model_path + '.trgvocab') self.attention_dialogue = AttentionDialogue.load_spec( model_path + '.spec', self.XP) serializers.load_hdf5(model_path + '.weights', self.attention_dialogue) trace('generating translation ...') generated = 0 with open(self.test_target, 'w') as fp: for src_batch in gens.batch(gens.word_list(self.source), self.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = self.forward_implement(src_batch, None, src_vocab, trg_vocab, self.attention_dialogue, False, self.generation_limit) source_cuont = 0 for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] print("src : " + "".join(src_batch[source_cuont]).replace("</s>", "")) print('hyp : ' + ''.join(hyp)) print(' '.join(hyp), file=fp) source_cuont = source_cuont + 1 generated += K trace('finished.')
def test(self): """ Test method You have to parepare the train model """ trace("loading model ...") prefix = self.model model_path = APP_ROOT + "/model/" + prefix src_vocab = Vocabulary.load(model_path + ".srcvocab") trg_vocab = Vocabulary.load(model_path + ".trgvocab") self.attention_dialogue = AttentionDialogue.load_spec(model_path + ".spec", self.XP) serializers.load_hdf5(model_path + ".weights", self.attention_dialogue) trace("generating translation ...") generated = 0 with open(self.test_target, "w") as fp: for src_batch in gens.batch(gens.word_list(self.source), self.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace("sample %8d - %8d ..." % (generated + 1, generated + K)) hyp_batch = self.forward_implement( src_batch, None, src_vocab, trg_vocab, self.attention_dialogue, False, self.generation_limit ) source_cuont = 0 for hyp in hyp_batch: hyp.append("</s>") hyp = hyp[: hyp.index("</s>")] print("src : " + "".join(src_batch[source_cuont]).replace("</s>", "")) print("hyp : " + "".join(hyp)) print(" ".join(hyp), file=fp) source_cuont = source_cuont + 1 generated += K trace("finished.")
def test_model(args): trace('loading model ...') model = EncoderDecoderModel.load(args.model) trace('generating translation ...') generated = 0 with open(args.target, 'w') as fp: for src_batch in gens.batch(gens.word_list(args.source), args.minibatch): src_batch = fill_batch(src_batch) K = len(src_batch) trace('sample %8d - %8d ...' % (generated + 1, generated + K)) hyp_batch = model.predict(src_batch, args.generation_limit) for hyp in hyp_batch: hyp.append('</s>') hyp = hyp[:hyp.index('</s>')] print(' '.join(hyp), file=fp) generated += K trace('finished.')