def test(self):
        trace('loading model ...')
        src_vocab = Vocabulary.load(self.model + '.srcvocab')
        trg_vocab = Vocabulary.load(self.model + '.trgvocab')
        encdec = EncoderDecoder.load_spec(self.model + '.spec')
        serializers.load_hdf5(self.model + '.weights', encdec)

        trace('generating translation ...')
        generated = 0

        with open(self.target, 'w') as fp:
            for src_batch in gens.batch(gens.word_list(self.source), self.minibatch):
                src_batch = fill_batch(src_batch)
                K = len(src_batch)

                trace('sample %8d - %8d ...' % (generated + 1, generated + K))
                hyp_batch = self.forward(src_batch, None, src_vocab, trg_vocab, encdec, False, self.generation_limit)

                source_cuont = 0
                for hyp in hyp_batch:
                    hyp.append('</s>')
                    hyp = hyp[:hyp.index('</s>')]
                    print("src : " + "".join(src_batch[source_cuont]).replace("</s>", ""))
                    print('hyp : ' +''.join(hyp))
                    print(' '.join(hyp), file=fp)
                    source_cuont = source_cuont + 1

                generated += K

        trace('finished.')
    def test(self):
        trace('loading model ...')
        src_vocab = Vocabulary.load(self.model + '.srcvocab')
        trg_vocab = Vocabulary.load(self.model + '.trgvocab')
        encdec = EncoderDecoder.load_spec(self.model + '.spec')
        serializers.load_hdf5(self.model + '.weights', encdec)

        trace('generating translation ...')
        generated = 0

        with open(self.target, 'w') as fp:
            for src_batch in gens.batch(gens.word_list(self.source),
                                        self.minibatch):
                src_batch = fill_batch(src_batch)
                K = len(src_batch)

                trace('sample %8d - %8d ...' % (generated + 1, generated + K))
                hyp_batch = self.forward(src_batch, None, src_vocab, trg_vocab,
                                         encdec, False, self.generation_limit)

                source_cuont = 0
                for hyp in hyp_batch:
                    hyp.append('</s>')
                    hyp = hyp[:hyp.index('</s>')]
                    print("src : " +
                          "".join(src_batch[source_cuont]).replace("</s>", ""))
                    print('hyp : ' + ''.join(hyp))
                    print(' '.join(hyp), file=fp)
                    source_cuont = source_cuont + 1

                generated += K

        trace('finished.')
Example #3
0
def test(args):
    trace('loading model ...')
    word_vocab = Vocabulary.load(args.model + '.words')
    phrase_vocab = Vocabulary.load(args.model + '.phrases')
    semiterminal_vocab = Vocabulary.load(args.model + '.semiterminals')
    parser = Parser.load_spec(args.model + '.spec')
    if args.use_gpu:
        parser.to_gpu()
    serializers.load_hdf5(args.model + '.weights', parser)

    embed_cache = {}
    parser.reset()

    trace('generating parse trees ...')
    with open(args.source) as fp:
        for l in fp:
            word_list = to_vram_words(convert_word_list(l.split(), word_vocab))
            tree = combine_xbar(
                restore_labels(
                    parser.forward(word_list, None, args.unary_limit,
                                   embed_cache), phrase_vocab,
                    semiterminal_vocab))
            print('( ' + tree_to_string(tree) + ' )')

    trace('finished.')
Example #4
0
 def load(filename):
     self = AttentionalTranslationModel()
     with ModelFile(filename) as fp:
         self.__src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_embed = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xi)
         fp.read_linear(self.__model.w_ia)
         fp.read_linear(self.__model.w_aa)
         fp.read_linear(self.__model.w_ib)
         fp.read_linear(self.__model.w_bb)
         fp.read_linear(self.__model.w_aw)
         fp.read_linear(self.__model.w_bw)
         fp.read_linear(self.__model.w_pw)
         fp.read_linear(self.__model.w_we)
         fp.read_linear(self.__model.w_ap)
         fp.read_linear(self.__model.w_bp)
         fp.read_embed(self.__model.w_yp)
         fp.read_linear(self.__model.w_pp)
         fp.read_linear(self.__model.w_cp)
         fp.read_linear(self.__model.w_dp)
         fp.read_linear(self.__model.w_py)
         wrapper.end_model_access(self.__model)
     return self
Example #5
0
def test(args):
  trace('loading model ...')
  word_vocab = Vocabulary.load(args.model + '.words')
  phrase_vocab = Vocabulary.load(args.model + '.phrases')
  semiterminal_vocab = Vocabulary.load(args.model + '.semiterminals')
  parser = Parser.load_spec(args.model + '.spec')
  if args.use_gpu:
    parser.to_gpu()
  serializers.load_hdf5(args.model + '.weights', parser)

  embed_cache = {}
  parser.reset()

  trace('generating parse trees ...')
  with open(args.source) as fp:
    for l in fp:
      word_list = to_vram_words(convert_word_list(l.split(), word_vocab))
      tree = combine_xbar(
          restore_labels(
              parser.forward(word_list, None, args.unary_limit, embed_cache),
              phrase_vocab,
              semiterminal_vocab))
      print('( ' + tree_to_string(tree) + ' )')

  trace('finished.')
Example #6
0
def test(args):
  trace('loading model ...')
  src_vocab = Vocabulary.load(args.model + '.srcvocab')
  trg_vocab = Vocabulary.load(args.model + '.trgvocab')
  attmt = AttentionMT.load_spec(args.model + '.spec')
  if args.use_gpu:
    attmt.to_gpu()
  serializers.load_hdf5(args.model + '.weights', attmt)
  
  trace('generating translation ...')
  generated = 0

  with open(args.target, 'w') as fp:
    for src_batch in gens.batch(gens.word_list(args.source), args.minibatch):
      src_batch = fill_batch(src_batch)
      K = len(src_batch)

      trace('sample %8d - %8d ...' % (generated + 1, generated + K))
      hyp_batch = forward(src_batch, None, src_vocab, trg_vocab, attmt, False, args.generation_limit)

      for hyp in hyp_batch:
        hyp.append('</s>')
        hyp = hyp[:hyp.index('</s>')]
        print(' '.join(hyp), file=fp)

      generated += K

  trace('finished.')
 def load(filename):
     self = AttentionalTranslationModel()
     with ModelFile(filename) as fp:
         self.__src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_embed = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xi)
         fp.read_linear(self.__model.w_ia)
         fp.read_linear(self.__model.w_aa)
         fp.read_linear(self.__model.w_ib)
         fp.read_linear(self.__model.w_bb)
         fp.read_linear(self.__model.w_aw)
         fp.read_linear(self.__model.w_bw)
         fp.read_linear(self.__model.w_pw)
         fp.read_linear(self.__model.w_we)
         fp.read_linear(self.__model.w_ap)
         fp.read_linear(self.__model.w_bp)
         fp.read_embed(self.__model.w_yp)
         fp.read_linear(self.__model.w_pp)
         fp.read_linear(self.__model.w_cp)
         fp.read_linear(self.__model.w_dp)
         fp.read_linear(self.__model.w_py)
         wrapper.end_model_access(self.__model)
     return self
 def __predict_sentence(self, src_batch):
     dialogue = EncoderDecoderModelForwardSlack(self.parameter)
     src_vocab = Vocabulary.load(self.model_name + '.srcvocab')
     trg_vocab = Vocabulary.load(self.model_name + '.trgvocab')
     model = EncoderDecoder.load_spec(self.model_name + '.spec')
     serializers.load_hdf5(dialogue.model + '.weights', model)
     hyp_batch = dialogue.forward(src_batch, None, src_vocab, trg_vocab, model, False, self.generation_limit)
     return hyp_batch
Example #9
0
 def __predict_sentence(self, src_batch):
     dialogue = EncoderDecoderModelForwardSlack(self.parameter)
     src_vocab = Vocabulary.load(self.model_name + '.srcvocab')
     trg_vocab = Vocabulary.load(self.model_name + '.trgvocab')
     model = EncoderDecoder.load_spec(self.model_name + '.spec')
     serializers.load_hdf5(dialogue.model + '.weights', model)
     hyp_batch = dialogue.forward(src_batch, None, src_vocab, trg_vocab,
                                  model, False, self.generation_limit)
     return hyp_batch
Example #10
0
def train(args):
    if os.path.exists("./model/vocab.bin"):
        src_vocab = Vocabulary.load("./model/vocab.bin")
    else:
        src_vocab = Vocabulary.new(gens.word_list(args.source), args.n_vocab)
        src_vocab.save('./model/vocab.bin')
    if os.path.exists("./model/tag.bin"):
        trg_tag = Vocabulary.load("./model/tag.bin")
    else:
        trg_tag = Vocabulary.new(gens.word_list(args.target), args.n_tag)
        trg_tag.save('./model/tag.bin')
    print("vocab_len:{}".format(src_vocab.__len__))
    print("tag_len:{}".format(trg_tag.__len__))
    encdec = BiEncDecLSTM(args.n_vocab, args.layer, args.embed, args.hidden,
                          args.n_tag)
    optimizer = optimizers.Adam()
    optimizer.setup(encdec)

    for e_i in range(args.epoch):
        tt_list = [[src_vocab.stoi(char) for char in char_arr]
                   for char_arr in gens.word_list(args.source_tr)]
        tag_list = [
            trg_tag.stoi(tag[0]) for tag in gens.word_list(args.target_tr)
        ]
        print("{}:{}".format(len(tt_list), len(tag_list)))
        assert len(tt_list) == len(tag_list)
        ind_arr = [ri for ri in range(len(tt_list))]
        random.shuffle(ind_arr)
        tt_now = (tt_list[ri] for ri in ind_arr)
        tag_now = (tag_list[ri] for ri in ind_arr)
        tt_gen = gens.batch(tt_now, args.batchsize)
        tag_gen = gens.batch(tag_now, args.batchsize)

        for tt, tag in zip(tt_gen, tag_gen):
            y_ws = encdec(tt)

            teac_arr = [src_vocab.itos(t) for t in tt[0]]
            pred_arr = [trg_tag.itos(y_each.data.argmax(0)) for y_each in y_ws]
            print("teach:{}:{}:{}".format(teac_arr, trg_tag.itos(tag[0]),
                                          pred_arr[0]))
            tag = xp.array(tag, dtype=xp.int32)
            loss = F.softmax_cross_entropy(y_ws, tag)

            encdec.cleargrads()
            loss.backward()
            optimizer.update()

            # loss.backward()
            # optimizer.target.cleargrads()
            # loss.backward()
            # loss.unchain_backward()
            # optimizer.update()

        serializers.save_npz('./model/attn_tag_model_{}.npz'.format(e_i),
                             encdec)
Example #11
0
  def __init__(self, args):
    trace('loading model ...')
    self.args = args
    self.src_vocab = Vocabulary.load(args.model + '.srcvocab')
    self.trg_vocab = Vocabulary.load(args.model + '.trgvocab')
    self.encdec = EncoderDecoder.load_spec(args.model + '.spec')
    if args.use_gpu:
      self.encdec.to_gpu()
    serializers.load_hdf5(args.model + '.weights', self.encdec)

    trace('generating translation ...')
 def __predict_sentence(self, src_batch):
     """
     predict sentence
     :param src_batch: get the source sentence
     :return:
     """
     dialogue = EncoderDecoderModelAttention(self.parameter)
     src_vocab = Vocabulary.load(self.model_name + '.srcvocab')
     trg_vocab = Vocabulary.load(self.model_name + '.trgvocab')
     model = AttentionDialogue.load_spec(self.model_name + '.spec', self.XP)
     serializers.load_hdf5(self.model_name + '.weights', model)
     hyp_batch = dialogue.forward_implement(src_batch, None, src_vocab, trg_vocab, model, False, self.generation_limit)
     return hyp_batch
 def __predict_sentence(self, src_batch):
     """
     predict sentence
     :param src_batch: get the source sentence
     :return:
     """
     dialogue = EncoderDecoderModelAttention(self.parameter)
     src_vocab = Vocabulary.load(self.model_name + '.srcvocab')
     trg_vocab = Vocabulary.load(self.model_name + '.trgvocab')
     model = AttentionDialogue.load_spec(self.model_name + '.spec', self.XP)
     serializers.load_hdf5(self.model_name + '.weights', model)
     hyp_batch = dialogue.forward_implement(src_batch, None, src_vocab,
                                            trg_vocab, model, False,
                                            self.generation_limit)
     return hyp_batch
 def load(self, filename):
     with ModelFile(filename) as fp:
         self.src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.n_embed = int(fp.read())
         self.n_hidden = int(fp.read())
         self.make_model()
         wrapper.begin_model_access(self.model)
         fp.read_embed(self.model.weight_xi)
         fp.read_linear(self.model.weight_ip)
         fp.read_linear(self.model.weight_pp)
         fp.read_linear(self.model.weight_pq)
         fp.read_linear(self.model.weight_qj)
         fp.read_linear(self.model.weight_jy)
         fp.read_embed(self.model.weight_yq)
         fp.read_linear(self.model.weight_qq)
         wrapper.end_model_access(self.model)
     return self
 def load(self, filename):
     with ModelFile(filename) as fp:
         self.src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.n_embed = int(fp.read())
         self.n_hidden = int(fp.read())
         self.make_model()
         wrapper.begin_model_access(self.model)
         fp.read_embed(self.model.weight_xi)
         fp.read_linear(self.model.weight_ip)
         fp.read_linear(self.model.weight_pp)
         fp.read_linear(self.model.weight_pq)
         fp.read_linear(self.model.weight_qj)
         fp.read_linear(self.model.weight_jy)
         fp.read_embed(self.model.weight_yq)
         fp.read_linear(self.model.weight_qq)
         wrapper.end_model_access(self.model)
     return self
Example #16
0
def test(args, epoch):
    model_name = "./model/attn_tag_model_{}.npz".format(epoch)
    encdec = BiEncDecLSTM(args.n_vocab, args.layer, args.embed, args.hidden,
                          args.n_tag)
    serializers.load_npz(model_name, encdec)
    src_vocab = Vocabulary.load("./model/vocab.bin")
    trg_tag = Vocabulary.load("./model/tag.bin")
    tt_now = ([src_vocab.stoi(char) for char in char_arr]
              for char_arr in gens.word_list(args.source_te))
    tag_now = (trg_tag.stoi(tag[0]) for tag in gens.word_list(args.target_te))
    tt_gen = gens.batch(tt_now, args.batchsize)
    tag_gen = gens.batch(tag_now, args.batchsize)
    correct_num = 0
    wrong_num = 0
    fw = codecs.open("./output/result_attn_tw{}.csv".format(epoch),
                     "w",
                     encoding="utf-8")
    fw.write("台詞,教師キャラ,予測キャラ,予測値,›単語\n")
    for tt, tag in zip(tt_gen, tag_gen):
        y, att_w = encdec.callAndAtt(tt)
        max_y = [
            max(
                F.softmax(F.reshape(y_each.data,
                                    (1, len(y_each.data)))).data[0])
            for y_each in y
        ]
        y = [y_each.data.argmax(0) for y_each in y]
        for tt_e, y_e, tag_e, max_y_e, att_w_e in zip(tt, y, tag, max_y,
                                                      att_w):
            txt = ",".join([src_vocab.itos(id) for id in tt_e])
            tag_e = trg_tag.itos(tag_e)
            y_e = trg_tag.itos(y_e)
            att_ind = att_w_e.data.argmax()
            most_word = src_vocab.itos(tt_e[att_ind])
            fw.write("{}:{}:{}:{}:{}\n".format(txt, tag_e, y_e, max_y_e,
                                               most_word))
        correct_num += len([1 for y_e, tag_e in zip(y, tag) if y_e == tag_e])
        wrong_num += len([1 for y_e, tag_e in zip(y, tag) if y_e != tag_e])
    print("epoch:{}".format(epoch))
    print(" correct:{}".format(correct_num))
    print(" wrong:{}".format(wrong_num))
    fw.write("correct{}\n".format(correct_num))
    fw.write("wrong:{}\n".format(wrong_num))
    fw.close()
Example #17
0
 def load(filename):
     self = EncoderDecoderModel()
     with ModelFile(filename) as fp:
         self.__src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_embed = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xi)
         fp.read_linear(self.__model.w_ip)
         fp.read_linear(self.__model.w_pp)
         fp.read_linear(self.__model.w_pq)
         fp.read_linear(self.__model.w_qj)
         fp.read_linear(self.__model.w_jy)
         fp.read_embed(self.__model.w_yq)
         fp.read_linear(self.__model.w_qq)
         wrapper.end_model_access(self.__model)
     return self
    def test(self):
        """
        Test method
        You have to parepare the train model
        """
        trace('loading model ...')
        prefix = self.model
        model_path = APP_ROOT + "/model/" + prefix
        src_vocab = Vocabulary.load(model_path + '.srcvocab')
        trg_vocab = Vocabulary.load(model_path + '.trgvocab')
        self.attention_dialogue = AttentionDialogue.load_spec(
            model_path + '.spec', self.XP)
        serializers.load_hdf5(model_path + '.weights', self.attention_dialogue)

        trace('generating translation ...')
        generated = 0

        with open(self.test_target, 'w') as fp:
            for src_batch in gens.batch(gens.word_list(self.source),
                                        self.minibatch):
                src_batch = fill_batch(src_batch)
                K = len(src_batch)

                trace('sample %8d - %8d ...' % (generated + 1, generated + K))
                hyp_batch = self.forward_implement(src_batch, None, src_vocab,
                                                   trg_vocab,
                                                   self.attention_dialogue,
                                                   False,
                                                   self.generation_limit)

                source_cuont = 0
                for hyp in hyp_batch:
                    hyp.append('</s>')
                    hyp = hyp[:hyp.index('</s>')]
                    print("src : " +
                          "".join(src_batch[source_cuont]).replace("</s>", ""))
                    print('hyp : ' + ''.join(hyp))
                    print(' '.join(hyp), file=fp)
                    source_cuont = source_cuont + 1

                generated += K

        trace('finished.')
Example #19
0
 def load(filename):
     self = EncoderDecoderModel()
     with ModelFile(filename) as fp:
         self.__src_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__trg_vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_embed = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xi)
         fp.read_linear(self.__model.w_ip)
         fp.read_linear(self.__model.w_pp)
         fp.read_linear(self.__model.w_pq)
         fp.read_linear(self.__model.w_qj)
         fp.read_linear(self.__model.w_jy)
         fp.read_embed(self.__model.w_yq)
         fp.read_linear(self.__model.w_qq)
         wrapper.end_model_access(self.__model)
     return self
Example #20
0
def test(args):
    trace('loading model ...')
    word_vocab = Vocabulary.load(args.model + '.words')
    phrase_vocab = Vocabulary.load(args.model + '.phrases')
    semi_vocab = Vocabulary.load(args.model + '.semiterminals')
    parser = Parser.load_spec(args.model + '.spec')
    if USE_GPU:
        parser.to_gpu()
    serializers.load_hdf5(args.model + '.weights', parser)

    trace('generating parse trees ...')
    with open(args.source) as fp:
        for l in fp:
            word_list = convert_word_list(l.split(), word_vocab)
            tree = restore_labels(
                parser.forward(word_list, None, args.unary_limit),
                phrase_vocab, semi_vocab)
            print('( ' + tree_to_string(tree) + ' )')

    trace('finished.')
Example #21
0
def test(args):

    trace('loading model ...')
    src_vocab = Vocabulary.load(args.model + '.srcvocab')
    trg_vocab = Vocabulary.load(args.model + '.trgvocab')
    encdec = EncoderDecoder.load_spec(args.model + '.spec')
    if args.use_gpu:
        encdec.to_gpu()
    serializers.load_hdf5(args.model + '.weights', encdec)

    trace('generating translation ...')
    generated = 0

    temp = gens.to_words(args.target)
    # temp.append("</s>")
    src_batch = []
    src_batch.append(temp)
    # src_batch = [['私は', '太郎', 'です', '(´', 'ー', '`*)', 'ウンウン', '</s>']]
    src_batch = fill_batch(src_batch)
    print("src_batch:", src_batch)
    K = len(src_batch)

    trace('sample %8d - %8d ...' % (generated + 1, generated + K))
    print("question:")
    for srp in src_batch:
        srp.append('</s>')
        srp = srp[:srp.index('</s>')]
        print(''.join(srp))

    hyp_batch = forward(src_batch, None, src_vocab, trg_vocab, encdec, False,
                        args.generation_limit)
    print("answser:")
    for hyp in hyp_batch:
        hyp.append('</s>')
        hyp = hyp[:hyp.index('</s>')]
        print(''.join(hyp))
    print("----------------")

    generated += K

    trace('finished.')
Example #22
0
 def load(filename):
     self = TransSegmentationModel()
     with ModelFile(filename) as fp:
         self.__vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_context = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xh)
         fp.read_linear(self.__model.w_hy)
         wrapper.end_model_access(self.__model)
     return self
 def load(filename):
     self = SegmentationModel()
     with ModelFile(filename) as fp:
         self.__vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_context = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xh)
         fp.read_linear(self.__model.w_hy)
         wrapper.end_model_access(self.__model)
     return self
Example #24
0
 def setCateg(self, args):
     categ_name = "./{}/categ_{}.bin".format(args.dataname, args.dataname)
     if os.path.exists(categ_name):
         categ_vocab = Vocabulary.load(categ_name)
     else:
         set_cat = set()
         [[set_cat.add(word) for word in word_arr] for word_arr in gens.word_list(args.category)]
         n_categ = len(set_cat) + 3
         print("n_categ:{}".format(n_categ))
         categ_vocab = Vocabulary.new(gens.word_list(args.category), n_categ)
         categ_vocab.save(categ_name)
     self.categ_vocab = categ_vocab
     return categ_vocab
Example #25
0
def test(args):
  trace('loading model ...')
  word_vocab = Vocabulary.load(args.model + '.words')
  phrase_vocab = Vocabulary.load(args.model + '.phrases')
  semi_vocab = Vocabulary.load(args.model + '.semiterminals')
  parser = Parser.load_spec(args.model + '.spec')
  if USE_GPU:
    parser.to_gpu()
  serializers.load_hdf5(args.model + '.weights', parser)

  trace('generating parse trees ...')
  with open(args.source) as fp:
    for l in fp:
      word_list = convert_word_list(l.split(), word_vocab)
      tree = restore_labels(
          parser.forward(word_list, None, args.unary_limit),
          phrase_vocab,
          semi_vocab
      )
      print('( ' + tree_to_string(tree) + ' )')

  trace('finished.')
    def test(self):
        """
        Test method
        You have to parepare the train model
        """
        trace("loading model ...")
        prefix = self.model
        model_path = APP_ROOT + "/model/" + prefix
        src_vocab = Vocabulary.load(model_path + ".srcvocab")
        trg_vocab = Vocabulary.load(model_path + ".trgvocab")
        self.attention_dialogue = AttentionDialogue.load_spec(model_path + ".spec", self.XP)
        serializers.load_hdf5(model_path + ".weights", self.attention_dialogue)

        trace("generating translation ...")
        generated = 0

        with open(self.test_target, "w") as fp:
            for src_batch in gens.batch(gens.word_list(self.source), self.minibatch):
                src_batch = fill_batch(src_batch)
                K = len(src_batch)

                trace("sample %8d - %8d ..." % (generated + 1, generated + K))
                hyp_batch = self.forward_implement(
                    src_batch, None, src_vocab, trg_vocab, self.attention_dialogue, False, self.generation_limit
                )

                source_cuont = 0
                for hyp in hyp_batch:
                    hyp.append("</s>")
                    hyp = hyp[: hyp.index("</s>")]
                    print("src : " + "".join(src_batch[source_cuont]).replace("</s>", ""))
                    print("hyp : " + "".join(hyp))
                    print(" ".join(hyp), file=fp)
                    source_cuont = source_cuont + 1

                generated += K

        trace("finished.")
Example #27
0
 def setVocab(self, args):
     vocab_name = "./{}/vocab_{}.bin".format(args.dataname, args.dataname)
     if os.path.exists(vocab_name):
         src_vocab = Vocabulary.load(vocab_name)
     else:
         set_vocab = set()
         [[set_vocab.add(word) for word in word_arr] for word_arr in gens.word_list(args.source)]
         n_vocab = len(set_vocab) + 3
         print("n_vocab:{}".format(n_vocab))
         print("arg_vocab:{}".format(args.n_vocab))
         src_vocab = Vocabulary.new(gens.word_list(args.source), args.n_vocab)
         src_vocab.save(vocab_name)
     self.vocab = src_vocab
     return src_vocab
 def load(filename):
     self = RNNSegmentationModel()
     with ModelFile(filename) as fp:
         self.__vocab = Vocabulary.load(fp.get_file_pointer())
         self.__n_embed = int(fp.read())
         self.__n_hidden = int(fp.read())
         self.__make_model()
         wrapper.begin_model_access(self.__model)
         fp.read_embed(self.__model.w_xe)
         fp.read_linear(self.__model.w_ea)
         fp.read_linear(self.__model.w_aa)
         fp.read_linear(self.__model.w_eb)
         fp.read_linear(self.__model.w_bb)
         fp.read_linear(self.__model.w_ay1)
         fp.read_linear(self.__model.w_by1)
         fp.read_linear(self.__model.w_ay2)
         fp.read_linear(self.__model.w_by2)
         wrapper.end_model_access(self.__model)
     return self
Example #29
0
def train(args):
    trace('making vocabularies ...')
    src_vocab = Vocabulary.new(gens.input_word_list(), args.vocab)
    trg_vocab = Vocabulary.new(gens.output_word_list(), args.vocab)
    trace('making model ...')
    encdec = EncoderDecoder(args.vocab, args.embed, args.hidden)

    if args.load_model != "":
        print("model load  %s ... " % (args.load_model))
        src_vocab = Vocabulary.load(args.load_model + '.srcvocab')
        trg_vocab = Vocabulary.load(args.load_model + '.trgvocab')
        encdec = EncoderDecoder.load_spec(args.load_model + '.spec')
        serializers.load_hdf5(args.load_model + '.weights', encdec)

    if args.use_gpu:
        encdec.to_gpu()

    for epoch in range(args.epoch):
        trace('epoch %d/%d: ' % (epoch + 1, args.epoch))
        trained = 0
        gen1 = gens.input_word_list()
        gen2 = gens.output_word_list()
        gen3 = gens.batch(
            gens.sorted_parallel(gen1, gen2, 100 * args.minibatch),
            args.minibatch)
        opt = optimizers.AdaGrad(lr=0.01)
        opt.setup(encdec)
        opt.add_hook(optimizer.GradientClipping(5))

        for src_batch, trg_batch in gen3:
            src_batch = fill_batch(src_batch)
            trg_batch = fill_batch(trg_batch)
            K = len(src_batch)
            hyp_batch, loss = forward(src_batch, trg_batch, src_vocab,
                                      trg_vocab, encdec, True, 0)
            loss.backward()
            opt.update()

            for k in range(K):
                trace('epoch %3d/%3d, sample %8d' %
                      (epoch + 1, args.epoch, trained + k + 1))
                trace(
                    '  src = ' +
                    ' '.join([x if x != '</s>' else '*'
                              for x in src_batch[k]]))
                trace(
                    '  trg = ' +
                    ' '.join([x if x != '</s>' else '*'
                              for x in trg_batch[k]]))
                trace(
                    '  hyp = ' +
                    ' '.join([x if x != '</s>' else '*'
                              for x in hyp_batch[k]]))

            trained += K

        if epoch % args.model_save_timing == 0:

            trace('saving model ...')
            prefix = args.model + '.%03.d' % (epoch + 1)
            src_vocab.save(prefix + '.srcvocab')
            trg_vocab.save(prefix + '.trgvocab')
            encdec.save_spec(prefix + '.spec')
            serializers.save_hdf5(prefix + '.weights', encdec)

    trace('finished.')
Example #30
0
from util.vocabulary import Vocabulary
from util import generators as gens
from util.controller import Controller
from util.wrapper import wrapper
from util.const import *

if __name__ == '__main__':
    args = parse_args()

    trace('initializing ...')
    wrapper = wrapper(args.gpu_id)
    wrapper.init()

    trace('loading vocab ...')
    #    src_vocab = Vocabulary.load(args.src_vocab)
    #    trg_vocab = Vocabulary.load(args.trg_vocab)
    src_vocab = Vocabulary.load(VOCAB_SRC)
    trg_vocab = Vocabulary.load(VOCAB_TRG)

    controller = Controller(args.folder_name)

    if args.mode == 'train':
        controller.train_model(BasicEncoderDecoderModel, src_vocab, trg_vocab,
                               args)
    elif args.mode == 'dev':
        controller.dev_model(BasicEncoderDecoderModel, src_vocab, trg_vocab,
                             args)
    elif args.mode == 'test':
        controller.test_model(BasicEncoderDecoderModel, src_vocab, trg_vocab,
                              args)