Exemplo n.º 1
0
 def __init__(self,
              numchars=256,
              charembdim=50,
              charemb=None,
              charinnerdim=100,
              numwords=1000,
              wordembdim=100,
              wordemb=None,
              wordinnerdim=200,
              maskid=None,
              bidir=False,
              returnall=False,
              **kw):
     # char level inits
     if charemb is None:
         charemb = VectorEmbed(indim=numchars, dim=charembdim)
     else:
         charemb = charemb
         charembdim = charemb.outdim
     if not issequence(charinnerdim):
         charinnerdim = [charinnerdim]
     charlayers, lastchardim = MakeRNU.make(charembdim,
                                            charinnerdim,
                                            bidir=bidir)
     charenc = SeqEncoder(charemb,
                          *charlayers).maskoptions(maskid, MaskMode.AUTO)
     # word level inits
     if wordemb is None:
         wordemb = VectorEmbed(indim=numwords, dim=wordembdim)
     elif wordemb is False:
         wordemb = None
         wordembdim = 0
     else:
         wordemb = wordemb
         wordembdim = wordemb.outdim
     if not issequence(wordinnerdim):
         wordinnerdim = [wordinnerdim]
     wordlayers, outdim = MakeRNU.make(wordembdim + lastchardim,
                                       wordinnerdim,
                                       bidir=bidir)
     wordenc = SeqEncoder(None, *wordlayers).maskoptions(MaskMode.NONE)
     if returnall:
         wordenc.all_outputs()
     self.outdim = outdim
     super(WordCharSentEnc, self).__init__(l1enc=charenc,
                                           l2emb=wordemb,
                                           l2enc=wordenc,
                                           maskid=maskid)
Exemplo n.º 2
0
 def __init__(self,
              indim=400,
              inpembdim=50,
              inpemb=None,
              innerdim=100,
              maskid=None,
              bidir=False,
              **kw):
     if inpemb is None:
         if inpembdim is None:
             inpemb = IdxToOneHot(indim)
             inpembdim = indim
         else:
             inpemb = VectorEmbed(indim=indim, dim=inpembdim)
     else:
         inpembdim = inpemb.outdim
     lastdim = inpembdim
     if not issequence(innerdim):  # single encoder
         innerdim = [innerdim]
     rnns = []
     for innerdimi in innerdim:
         if not issequence(innerdimi):  # one layer in encoder
             innerdimi = [innerdimi]
         rnn, lastdim = MakeRNU.make(lastdim, innerdimi, bidir=bidir)
         rnns.append(rnn)
     self.outdim = lastdim
     super(SimpleSeqStar2Vec, self).__init__(inpemb,
                                             *rnns,
                                             maskid=maskid,
                                             **kw)
Exemplo n.º 3
0
 def makernu(inpembdim,
             innerdim,
             bidir=False,
             zoneout=False,
             dropout_h=False,
             dropout_in=False):
     return MakeRNU.make(inpembdim,
                         innerdim,
                         bidir=bidir,
                         zoneout=zoneout,
                         dropout_h=dropout_h,
                         dropout_in=dropout_in)
Exemplo n.º 4
0
 def __init__(self,
              indim=400,
              outdim=50,
              inpembdim=50,
              outembdim=50,
              innerdim=100,
              **kw):
     self.inpemb = VectorEmbed(indim=indim, dim=inpembdim)
     self.outemb = VectorEmbed(indim=outdim, dim=outembdim)
     self.rnn = []
     if not issequence(innerdim):
         innerdim = [innerdim]
     innerdim = [inpembdim + outembdim] + innerdim
     self.rnn = MakeRNU.fromdims(innerdim)[0]
     super(SimpleSeqTransDec, self).__init__(self.inpemb,
                                             self.outemb,
                                             *self.rnn,
                                             smodim=innerdim[-1],
                                             outdim=outdim,
                                             **kw)
Exemplo n.º 5
0
 def __init__(self,
              indim=400,
              embdim=50,
              inpemb=None,
              innerdim=100,
              outdim=50,
              rnu=GRU,
              **kw):
     if inpemb is None:
         emb = VectorEmbed(indim=indim, dim=embdim)
     else:
         emb = inpemb
         embdim = emb.outdim
     if not issequence(innerdim):
         innerdim = [innerdim]
     innerdim = [embdim] + innerdim
     rnn, _ = MakeRNU.fromdims(innerdim, rnu=rnu)
     smo = Lin(indim=innerdim[-1], dim=outdim)
     super(SimpleSeqTrans, self).__init__(emb, *(rnn + [smo, Softmax()]),
                                          **kw)
Exemplo n.º 6
0
 def __init__(self,
              indim=400,
              embdim=50,
              inpemb=None,
              innerdim=100,
              outdim=50,
              rnu=GRU,
              **kw):
     if inpemb is None:
         self.emb = VectorEmbed(indim=indim, dim=embdim)
     else:
         self.emb = inpemb
         embdim = self.emb.outdim
     if not issequence(innerdim):
         innerdim = [innerdim]
     innerdim = [embdim] + innerdim
     self.rnn = MakeRNU.fromdims(innerdim, rnu=rnu)[0]
     super(SimpleSeqTransducer, self).__init__(self.emb,
                                               *self.rnn,
                                               smodim=innerdim[-1],
                                               outdim=outdim,
                                               **kw)
Exemplo n.º 7
0
    def __init__(self,
                 inpvocsize=None,
                 inpembdim=None,
                 inpemb=None,
                 inpencinnerdim=None,
                 bidir=False,
                 maskid=None,
                 dropout=False,
                 rnu=GRU,
                 inpencoder=None,
                 memvocsize=None,
                 memembdim=None,
                 memembmat=None,
                 memencinnerdim=None,
                 memencoder=None,
                 inp_att_dist=CosineDistance(),
                 mem_att_dist=CosineDistance(),
                 inp_attention=None,
                 mem_attention=None,
                 coredims=None,
                 corernu=GRU,
                 core=None,
                 explicit_interface=False,
                 scalaraggdim=None,
                 write_value_dim=None,
                 nsteps=100,
                 posvecdim=None,
                 mem_pos_repr=None,
                 inp_pos_repr=None,
                 inp_addr_extractor=None,
                 mem_addr_extractor=None,
                 write_addr_extractor=None,
                 write_addr_generator=None,
                 write_addr_dist=CosineDistance(),
                 write_value_generator=None,
                 write_value_extractor=None,
                 mem_erase_generator=None,
                 mem_change_generator=None,
                 memsampler=None,
                 memsamplemethod=None,
                 memsampletemp=0.3,
                 **kw):

        # INPUT ENCODING
        if inpencoder is None:
            inpencoder = SeqEncoder.RNN(indim=inpvocsize,
                                        inpembdim=inpembdim,
                                        inpemb=inpemb,
                                        innerdim=inpencinnerdim,
                                        bidir=bidir,
                                        maskid=maskid,
                                        dropout_in=dropout,
                                        dropout_h=dropout,
                                        rnu=rnu).all_outputs()
            lastinpdim = inpencinnerdim if not issequence(
                inpencinnerdim) else inpencinnerdim[-1]
        else:
            lastinpdim = inpencoder.block.layers[-1].innerdim

        # MEMORY ENCODING
        if memembmat is None:
            memembmat = param((memvocsize, memembdim),
                              name="memembmat").glorotuniform()
        if memencoder is None:
            memencoder = SeqEncoder.RNN(inpemb=False,
                                        innerdim=memencinnerdim,
                                        bidir=bidir,
                                        dropout_in=dropout,
                                        dropout_h=dropout,
                                        rnu=rnu,
                                        inpembdim=memembdim).all_outputs()
            lastmemdim = memencinnerdim if not issequence(
                memencinnerdim) else memencinnerdim[-1]
        else:
            lastmemdim = memencoder.block.layers[-1].innerdim

        # POSITION VECTORS
        if posvecdim is not None and inp_pos_repr is None:
            inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)
        if posvecdim is not None and mem_pos_repr is None:
            mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout)

        xtra_dim = posvecdim if posvecdim is not None else 0
        # CORE RNN - THE THINKER
        if core is None:
            corelayers, _ = MakeRNU.fromdims(
                [lastinpdim + lastmemdim + xtra_dim * 2] + coredims,
                rnu=corernu,
                dropout_in=dropout,
                dropout_h=dropout,
                param_init_states=True)
            core = RecStack(*corelayers)

        lastcoredim = core.get_statespec()[-1][0][1][0]

        # ATTENTIONS
        if mem_attention is None:
            mem_attention = Attention(mem_att_dist)
        if inp_attention is None:
            inp_attention = Attention(inp_att_dist)
        if write_addr_generator is None:
            write_addr_generator = AttGen(write_addr_dist)

        # WRITE VALUE
        if write_value_generator is None:
            write_value_generator = WriteValGenerator(write_value_dim,
                                                      memvocsize,
                                                      dropout=dropout)

        # MEMORY SAMPLER
        if memsampler is not None:
            assert (memsamplemethod is None)
        if memsamplemethod is not None:
            assert (memsampler is None)
            memsampler = GumbelSoftmax(temperature=memsampletemp)

        ################ STATE INTERFACES #################

        if not explicit_interface:
            if inp_addr_extractor is None:
                inp_addr_extractor = Forward(lastcoredim,
                                             lastinpdim + xtra_dim,
                                             dropout=dropout)
            if mem_addr_extractor is None:
                inp_addr_extractor = Forward(lastcoredim,
                                             lastmemdim + xtra_dim,
                                             dropout=dropout)

            # WRITE INTERFACE
            if write_addr_extractor is None:
                write_addr_extractor = Forward(lastcoredim,
                                               lastmemdim + xtra_dim,
                                               dropout=dropout)
            if write_value_extractor is None:
                write_value_extractor = Forward(lastcoredim,
                                                write_value_dim,
                                                dropout=dropout)

            # MEM UPDATE INTERFACE
            if mem_erase_generator is None:
                mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim)
            if mem_change_generator is None:
                mem_change_generator = StateToScalar(lastcoredim, scalaraggdim)
        else:
            inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \
            write_value_extractor, mem_erase_generator, mem_change_generator = \
                make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim,
                                    lastmemdim + xtra_dim, write_value_dim, 1, 1)

        super(SimpleBulkNN,
              self).__init__(inpencoder=inpencoder,
                             memembmat=memembmat,
                             memencoder=memencoder,
                             inp_attention=inp_attention,
                             mem_attention=mem_attention,
                             core=core,
                             memsampler=memsampler,
                             nsteps=nsteps,
                             inp_addr_extractor=inp_addr_extractor,
                             mem_addr_extractor=mem_addr_extractor,
                             write_addr_extractor=write_addr_extractor,
                             write_addr_generator=write_addr_generator,
                             mem_erase_generator=mem_erase_generator,
                             mem_change_generator=mem_change_generator,
                             write_value_generator=write_value_generator,
                             write_value_extractor=write_value_extractor,
                             inp_pos_repr=inp_pos_repr,
                             mem_pos_repr=mem_pos_repr,
                             **kw)
Exemplo n.º 8
0
 def makernu(inpembdim, innerdim, bidir=False):
     return MakeRNU.make(inpembdim, innerdim, bidir=bidir)
Exemplo n.º 9
0
 def getrnnfrominnerdim(self, innerdim, rnu=GRU):
     assert (len(innerdim) >= 2)
     initdim = innerdim[0]
     otherdim = innerdim[1:]
     return MakeRNU.make(initdim, otherdim, rnu=rnu)[0]
Exemplo n.º 10
0
 def makernu(inpembdim, innerdim, bidir=False):
     return MakeRNU.make(inpembdim, innerdim, bidir=bidir)
Exemplo n.º 11
0
 def getrnnfrominnerdim(self, innerdim, rnu=GRU):
     assert(len(innerdim) >= 2)
     initdim = innerdim[0]
     otherdim = innerdim[1:]
     return MakeRNU.make(initdim, otherdim, rnu=rnu)[0]