head = '%(asctime)-15s %(message)s'
logging.basicConfig(level=logging.DEBUG, format=head)
segmenter = SpaceSegmenter()
corpus = SequenceCorpus()
corpus.build(open(sys.argv[1], 'r'), segmenter)
cell_num = corpus.cell_num()
problem = LanguageModelProblem(corpus)
batch_size = 32
data_train = BucketIter(problem, batch_size)
val_corpus = corpus.make(open(sys.argv[2], 'r'), segmenter)
val_problem = LanguageModelProblem(val_corpus)
data_val = BucketIter(val_problem, batch_size)
arch_param = LanguageModelArchParam(num_hidden=200,
num_embed=200,
num_lstm_layer=2,
cell_num=corpus.cell_num())
learning_param = LanguageModelLearnParam(num_epoch=25,
learning_rate=0.01,
momentum=0.0)
lm = LanguageModel(arch_param)
lm.train(data_train, data_val, learning_param)
logging.basicConfig(level=logging.DEBUG, format=head)
segmenter = SpaceSegmenter()
corpus = SequenceCorpus()
corpus.build(open(sys.argv[1], 'r'), segmenter)
cell_num = corpus.cell_num()
problem = LanguageModelProblem(corpus)
batch_size = 32
data_train = BucketIter(problem, batch_size)
val_corpus = corpus.make(open(sys.argv[2], 'r'), segmenter)
val_problem = LanguageModelProblem(val_corpus)
data_val = BucketIter(val_problem, batch_size)
arch_param = LanguageModelArchParam(
num_hidden= 200,
num_embed= 200,
num_lstm_layer= 2,
cell_num = corpus.cell_num()
)
learning_param = LanguageModelLearnParam(
num_epoch=25,learning_rate=0.01, momentum=0.0
)
