def train_model(args):
train_begin = time.time()
trace('making vocaburaries ...')
vocab = Vocabulary.new(gens.letter_list(args.corpus), args.vocab)
trace('begin training ...')
model = TransSegmentationModel.new(vocab, args.context, args.hidden, args.labels, args.eta)
for epoch in range(args.epoch):
epoch_beg = time.time()
trace('START epoch %d/%d: ' % (epoch + 1, args.epoch))
trained = 0
total_loss = 0
model.init_optimizer()
with open(args.corpus) as fp:
for text in fp:
word_list = text.split()
if not word_list:
continue
text = ' '.join(word_list)
letters = ''.join(word_list)
labels, accum_loss_f = model.train(text)
total_loss += accum_loss_f
trained += 1
hyp = make_hyp(letters, labels)
"""for 1sentence output
trace("accum_loss : %lf"% (accum_loss_f))
trace('epoch %d/%d: ' % (epoch + 1, args.epoch))
trace('trained %d: '% trained)
trace(text)
trace(hyp)
"""
"""
if trained % 100 == 0:
trace(' %8d' % trained)
"""
trace('FINISHED epoch %d/%d: ' % (epoch + 1, args.epoch))
trace('total_loss : %lf'%total_loss)
trace('saving model ...')
model.save(args.model + '.%03d' % (epoch + 1))
epoch_time = time.time() - epoch_beg
trace('elapsed_time/1epoch : %lf'%epoch_time)
trace('finished.')
elapsed_time = time.time() - train_begin
trace('train_time : %lf'%elapsed_time)
trace('')
def train_model(args):
train_begin = time.time()
trace('making vocaburaries ...')
vocab = Vocabulary.new(gens.letter_list(args.corpus), args.vocab)
trace('begin training ...')
model = TransSegmentationModel.new(vocab, args.context, args.hidden,
args.labels, args.eta)
for epoch in range(args.epoch):
epoch_beg = time.time()
trace('START epoch %d/%d: ' % (epoch + 1, args.epoch))
trained = 0
total_loss = 0
model.init_optimizer()
with open(args.corpus) as fp:
for text in fp:
word_list = text.split()
if not word_list:
continue
text = ' '.join(word_list)
letters = ''.join(word_list)
labels, accum_loss_f = model.train(text)
total_loss += accum_loss_f
trained += 1
hyp = make_hyp(letters, labels)
"""for 1sentence output
trace("accum_loss : %lf"% (accum_loss_f))
trace('epoch %d/%d: ' % (epoch + 1, args.epoch))
trace('trained %d: '% trained)
trace(text)
trace(hyp)
"""
"""
if trained % 100 == 0:
trace(' %8d' % trained)
"""
trace('FINISHED epoch %d/%d: ' % (epoch + 1, args.epoch))
trace('total_loss : %lf' % total_loss)
trace('saving model ...')
model.save(args.model + '.%03d' % (epoch + 1))
epoch_time = time.time() - epoch_beg
trace('elapsed_time/1epoch : %lf' % epoch_time)
trace('finished.')
elapsed_time = time.time() - train_begin
trace('train_time : %lf' % elapsed_time)
trace('')
def train_model(args):
trace('making vocabularies ...')
vocab = Vocabulary.new(gens.letter_list(args.corpus), args.vocab)
trace('start training ...')
model = RNNSegmentationModel.new(vocab, args.embed, args.hidden)
for epoch in range(args.epoch):
trace('epoch %d/%d: ' % (epoch + 1, args.epoch))
trained = 0
model.init_optimizer()
with open(args.corpus) as fp:
for text in fp:
word_list = text.split()
if not word_list:
continue
text = ' '.join(word_list)
letters = ''.join(word_list)
scores = model.train(text)
trained += 1
hyp = make_hyp(letters, scores)
trace(trained)
trace(text)
trace(hyp)
trace(' '.join('%+.1f' % x for x in scores))
if trained % 100 == 0:
trace(' %8d' % trained)
trace('saveing model ...')
model.save(args.model + '.%03d' % (epoch + 1))
trace('finished.')
def train_model(args):
trace("making vocabularies ...")
vocab = Vocabulary.new(gens.letter_list(args.corpus), args.vocab)
trace("start training ...")
model = SegmentationModel.new(vocab, args.context, args.hidden)
for epoch in range(args.epoch):
trace("epoch %d/%d: " % (epoch + 1, args.epoch))
trained = 0
model.init_optimizer()
with open(args.corpus) as fp:
for text in fp:
word_list = text.split()
if not word_list:
continue
text = " ".join(word_list)
letters = "".join(word_list)
scores = model.train(text)
trained += 1
hyp = make_hyp(letters, scores)
trace(trained)
trace(text)
trace(hyp)
trace(" ".join("%+.1f" % x for x in scores))
if trained % 100 == 0:
trace(" %8d" % trained)
trace("saveing model ...")
model.save(args.model + ".%03d" % (epoch + 1))
trace("finished.")
