#coding:utf-8 import sys reload(sys) sys.setdefaultencoding('utf-8') import os import dataloader as dd from keras.optimizers import * from keras.callbacks import * itokens, otokens = dd.MakeS2SDict('en2de.s2s.examples.txt', dict_file='en2de_word.txt') print('seq 1 words:', itokens.num()) print('seq 2 words:', otokens.num()) from transformer import Transformer, LRSchedulerPerStep d_model = 256 n_head = 4 s2s = Transformer(itokens, otokens, len_limit=500, d_model=d_model, d_inner_hid=1024, \ n_head=n_head, d_k=d_model//n_head, d_v=d_model//n_head, layers=3, dropout=0.1) lr_scheduler = LRSchedulerPerStep(d_model, 4000) model_saver = ModelCheckpoint('en2de.model.h5', monitor='ppl', save_best_only=True, save_weights_only=True) #s2s.model.summary()
import os, sys import dataloader as dd from keras.optimizers import * from keras.callbacks import * itokens, otokens = dd.MakeS2SDict('data/ara.txt', dict_file='data/ara_word.txt') Xtrain, Ytrain = dd.MakeS2SData('data/ara.txt', itokens, otokens, h5_file='data/ara.h5') Xvalid, Yvalid = dd.MakeS2SData('data/ara.txt', itokens, otokens, h5_file='data/ara.valid.h5') print('seq 1 words:', itokens.num()) print('seq 2 words:', otokens.num()) print('train shapes:', Xtrain.shape, Ytrain.shape) print('valid shapes:', Xvalid.shape, Yvalid.shape) ''' from rnn_s2s import RNNSeq2Seq s2s = RNNSeq2Seq(itokens,otokens, 256) s2s.compile('rmsprop') s2s.model.fit([Xtrain, Ytrain], None, batch_size=64, epochs=30, validation_data=([Xvalid, Yvalid], None)) ''' from transformer import Transformer, LRSchedulerPerStep, LRSchedulerPerEpoch d_model = 256 s2s = Transformer(itokens, otokens, len_limit=70, d_model=d_model, d_inner_hid=512, \
import os, sys import dataloader as dd from keras.optimizers import * from keras.callbacks import * itokens, otokens = dd.MakeS2SDict('data/en2de.s2s.txt', dict_file='data/en2de_word.txt') Xtrain, Ytrain = dd.MakeS2SData('data/en2de.s2s.txt', itokens, otokens, h5_file='data/en2de.h5') Xvalid, Yvalid = dd.MakeS2SData('data/en2de.s2s.valid.txt', itokens, otokens, h5_file='data/en2de.valid.h5') print('seq 1 words:', itokens.num()) print('seq 2 words:', otokens.num()) print('train shapes:', Xtrain.shape, Ytrain.shape) print('valid shapes:', Xvalid.shape, Yvalid.shape) ''' from rnn_s2s import RNNSeq2Seq s2s = RNNSeq2Seq(itokens,otokens, 256) s2s.compile('rmsprop') s2s.model.fit([Xtrain, Ytrain], None, batch_size=64, epochs=30, validation_data=([Xvalid, Yvalid], None)) ''' from transformer import Transformer, LRSchedulerPerStep, LRSchedulerPerEpoch d_model = 256 s2s = Transformer(itokens, otokens, len_limit=70, d_model=d_model, d_inner_hid=512, \ n_head=4, d_k=64, d_v=64, layers=2, dropout=0.1) mfile = 'models/en2de.model.h5' lr_scheduler = LRSchedulerPerStep(d_model, 4000) # there is a warning that it is slow, however, it's ok. #lr_scheduler = LRSchedulerPerEpoch(d_model, 4000, Xtrain.shape[0]/64) # this scheduler only update lr per epoch
#coding:utf-8 import sys reload(sys) sys.setdefaultencoding('utf-8') import os import dataloader as dd from keras.optimizers import * from keras.callbacks import * itokens, otokens = dd.MakeS2SDict('pinyin.corpus.txt', dict_file='pinyin_word.txt') print('seq 1 words:', itokens.num()) print('seq 2 words:', otokens.num()) from transformer import Transformer, LRSchedulerPerStep d_model = 256 n_head = 4 s2s = Transformer(itokens, otokens, len_limit=500, d_model=d_model, d_inner_hid=1024, \ n_head=n_head, d_k=d_model//n_head, d_v=d_model//n_head, layers=3, dropout=0.1) lr_scheduler = LRSchedulerPerStep(d_model, 4000) model_saver = ModelCheckpoint('pinyin.model.h5', monitor='ppl', save_best_only=True, save_weights_only=True)
import os, sys import dataloader as dd from tensorflow.keras.optimizers import * from tensorflow.keras.callbacks import * itokens, otokens = dd.MakeS2SDict('data/pinyin.corpus.examples.txt', dict_file='data/pinyin_word.txt') print('seq 1 words:', itokens.num()) print('seq 2 words:', otokens.num()) from transformer import Transformer, LRSchedulerPerStep d_model = 256 s2s = Transformer(itokens, otokens, len_limit=500, d_model=d_model, d_inner_hid=1024, \ n_head=4, layers=3, dropout=0.1) mfile = 'models/pinyin.model.h5' lr_scheduler = LRSchedulerPerStep(d_model, 4000) model_saver = ModelCheckpoint(mfile, monitor='ppl', save_best_only=True, save_weights_only=True) #s2s.model.summary() opt = Adam(0.001, 0.9, 0.98, epsilon=1e-9) s2s.compile(opt) try: s2s.model.load_weights(mfile) except: