out_file += "-" + str(epoch)
UF.trace("saving model to " + out_file + "...")
serializer = ModelSerializer(out_file)
serializer.save(model)
def onEpochUpdate(epoch_loss, prev_loss, epoch):
UF.trace("Train Loss:", float(prev_loss), "->", float(epoch_loss))
UF.trace("Train PPL:", math.exp(float(prev_loss)), "->", math.exp(float(epoch_loss)))
if dev_data is not None:
dev_loss = trainer.eval(dev_data, model)
UF.trace("Dev Loss:", float(dev_loss))
UF.trace("Dev PPL:", math.exp(float(dev_loss)))
# saving model
if args.save_models and (epoch + 1) % args.save_len == 0:
save_model(epoch)
def onTrainingFinish(epoch):
if not args.save_models or epoch % args.save_len != 0:
save_model(epoch)
if epoch == args.epoch:
UF.trace("training complete!")
""" Execute Training loop """
if args.one_epoch: one_epoch = True
else: one_epoch = False
trainer.train(train_data, model, args.epoch, onEpochStart, onBatchUpdate, onEpochUpdate, onTrainingFinish, one_epoch=one_epoch)
def onBatchUpdate(output, src, trg, trained, epoch, accum_loss):
if args.verbose:
report(output, src, trg, trained, epoch)
UF.trace("Trained %d: %f, col_size=%d" % (trained, accum_loss, len(trg[0])))
def save_model(epoch):
out_file = args.model_out
if args.save_models:
out_file += "-" + str(epoch)
UF.trace("saving model to " + out_file + "...")
serializer = ModelSerializer(out_file)
serializer.save(model)
def onEpochUpdate(epoch_loss, prev_loss, epoch):
UF.trace("Train Loss:", float(prev_loss), "->", float(epoch_loss))
UF.trace("Train PPL:", math.exp(float(prev_loss)), "->", math.exp(float(epoch_loss)))
# saving model
if args.save_models and (epoch + 1) % args.save_len == 0:
save_model(epoch)
def onTrainingFinish(epoch):
if not args.save_models or epoch % args.save_len != 0:
save_model(epoch)
UF.trace("training complete!")
""" Execute Training loop """
trainer.train(data, model, args.epoch, onEpochStart, onBatchUpdate, onEpochUpdate, onTrainingFinish)
