def setUp(self): criteriondim = 20 datadim = 20 innerdim = 30 batsize = 33 seqlen = 11 self.attgenshape = (batsize, seqlen) self.attconshape = (batsize, datadim) self.attgen = self.getattgenc(critdim=criteriondim, datadim=datadim, attdim=innerdim) self.attgenparams = self.getattgenparams() self.attcon = WeightedSumAttCon() self.att = Attention(self.attgen, self.attcon) self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32") self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def __init__(self, enclayers, declayers, attgen, attcon, decinnerdim, statetrans=None, vecout=False, inconcat=True, outconcat=False, **kw): enc = SeqEncoder(*enclayers)\ .with_outputs()\ .with_mask()\ .maskoptions(-1, MaskMode.AUTO, MaskSetMode.ZERO) smo = False if vecout else None dec = SeqDecoder(declayers, attention=Attention(attgen, attcon), innerdim=decinnerdim, inconcat=inconcat, softmaxoutblock=smo, outconcat=outconcat) super(SeqEncDecAtt, self).__init__(enc, dec, statetrans=statetrans, **kw)
def __init__(self, wordembdim=50, entembdim=200, innerdim=200, attdim=100, outdim=1e4, numwords=4e5, **kw): super(FBSeqSimpEncDecAtt, self).__init__(**kw) self.indim = wordembdim self.outdim = outdim self.wordembdim = wordembdim self.encinnerdim = innerdim self.decinnerdim = innerdim self.entembdim = entembdim self.wordencoder = WordEmbed(indim=numwords, outdim=self.wordembdim, trainfrac=1.0) self.rnn = RecStack( self.wordencoder, GRU(dim=self.wordembdim, innerdim=self.encinnerdim)) attgen = LinearGateAttentionGenerator(indim=self.encinnerdim + self.decinnerdim, attdim=attdim) attcon = WeightedSumAttCon() self.dec = SeqDecoder([ VectorEmbed(indim=self.outdim, dim=self.entembdim), GRU(dim=self.entembdim, innerdim=self.decinnerdim) ], attention=Attention(attgen, attcon), outconcat=True, inconcat=False, innerdim=self.encinnerdim + self.decinnerdim)
def setUp(self): vocsize = 10 innerdim = 50 encdim = 30 seqlen = 5 batsize = 77 self.att = Attention( LinearSumAttentionGenerator(indim=innerdim + encdim), WeightedSumAttCon()) self.decwatt = SeqDecoder([ IdxToOneHot(vocsize), GRU(dim=vocsize + encdim, innerdim=innerdim) ], inconcat=True, attention=self.att, innerdim=innerdim) self.decwoatt = SeqDecoder([ IdxToOneHot(vocsize), GRU(dim=vocsize + encdim, innerdim=innerdim) ], inconcat=True, innerdim=innerdim) self.attdata = np.random.random( (batsize, seqlen, encdim)).astype("float32") self.data = np.random.random((batsize, encdim)).astype("float32") self.seqdata = np.random.randint(0, vocsize, (batsize, seqlen)) self.predshape = (batsize, seqlen, vocsize)
def __init__(self, vocsize=25, outvocsize=25, encdim=200, innerdim=200, attdim=50, **kw): super(RewAttSumDecoder, self).__init__(**kw) self.rnn = SeqEncoder(IdxToOneHot(vocsize), GRU(dim=vocsize, innerdim=encdim)).all_outputs attgen = LinearGateAttentionGenerator(indim=innerdim+encdim, innerdim=attdim) attcon = WeightedSumAttCon() self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize+encdim, innerdim=innerdim)], inconcat=True, attention=Attention(attgen, attcon), innerdim=innerdim)
class DummyAttentionGeneratorConsumerTest(TestCase): def setUp(self): criteriondim = 30 datadim = 20 innerdim = 25 batsize = 33 seqlen = 11 self.attgenshape = (batsize, seqlen) self.attconshape = (batsize, datadim) self.attgenc = self.getattgenc() self.attgen = self.attgenc(indim=criteriondim + datadim, innerdim=innerdim) self.attgenparams = self.getattgenparams() self.attcon = WeightedSumAttCon() self.att = Attention(self.attgen, self.attcon) self.criterion_val = np.random.random( (batsize, criteriondim)).astype("float32") self.data_val = np.random.random( (batsize, seqlen, datadim)).astype("float32") def getattgenc(self): return LinearSumAttentionGenerator def getattgenparams(self): return {self.attgen.W} def test_generator_shape(self): pred = self.attgen.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attgenshape) def test_generator_param_prop(self): self.attgen.predict(self.criterion_val, self.data_val) allparams = self.attgen.output.allparams self.assertSetEqual(allparams, self.attgenparams) def test_consumer_shape(self): pred = self.att.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attconshape) def test_consumer_param_prop(self): self.att.predict(self.criterion_val, self.data_val) allparams = self.att.output.allparams self.assertSetEqual(allparams, self.attgenparams)
def __init__(self, vocsize=25, outvocsize=20, encdim=200, innerdim=200, attdim=50, **kw): super(FwdAttRNNEncDecoder, self).__init__(**kw) self.emb = IdxToOneHot(vocsize) attgen = LinearGateAttentionGenerator(indim=innerdim+vocsize, innerdim=attdim) attcon = SeqEncoder(None, GRU(dim=vocsize, innerdim=encdim)) self.dec = SeqDecoder([IdxToOneHot(outvocsize), GRU(dim=outvocsize, innerdim=innerdim)], outconcat=True, inconcat=False, attention=Attention(attgen, attcon), innerdim=innerdim+encdim)
class DummyAttentionGeneratorConsumerTest(TestCase): def setUp(self): criteriondim = 30 datadim = 20 innerdim = 25 batsize = 33 seqlen = 11 self.attgenshape = (batsize, seqlen) self.attconshape = (batsize, datadim) self.attgenc = self.getattgenc() self.attgen = self.attgenc(indim=criteriondim + datadim, innerdim=innerdim) self.attgenparams = self.getattgenparams() self.attcon = WeightedSumAttCon() self.att = Attention(self.attgen, self.attcon) self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32") self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32") def getattgenc(self): return LinearSumAttentionGenerator def getattgenparams(self): return {self.attgen.W} def test_generator_shape(self): pred = self.attgen.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attgenshape) def test_generator_param_prop(self): self.attgen.predict(self.criterion_val, self.data_val) allparams = self.attgen.output.allparams self.assertSetEqual(allparams, self.attgenparams) def test_consumer_shape(self): pred = self.att.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attconshape) def test_consumer_param_prop(self): self.att.predict(self.criterion_val, self.data_val) allparams = self.att.output.allparams self.assertSetEqual(allparams, self.attgenparams)
class DummyAttentionGeneratorConsumerTest(TestCase): def setUp(self): criteriondim = 20 datadim = 20 innerdim = 30 batsize = 33 seqlen = 11 self.attgenshape = (batsize, seqlen) self.attconshape = (batsize, datadim) self.attgen = self.getattgenc(critdim=criteriondim, datadim=datadim, attdim=innerdim) self.attgenparams = self.getattgenparams() self.attcon = WeightedSumAttCon() self.att = Attention(self.attgen, self.attcon) self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32") self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32") def getattgenc(self, critdim=None, datadim=None, attdim=None): return AttGen(DotDistance()) def getattgenparams(self): return set() def test_generator_shape(self): pred = self.attgen.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attgenshape) def test_generator_param_prop(self): _, outps = self.attgen.autobuild(self.criterion_val, self.data_val) allparams = outps[0].allparams self.assertSetEqual(allparams, self.attgenparams) def test_consumer_shape(self): pred = self.att.predict(self.criterion_val, self.data_val) self.assertEqual(pred.shape, self.attconshape) def test_consumer_param_prop(self): _, outps = self.att.autobuild(self.criterion_val, self.data_val) allparams = outps[0].allparams self.assertSetEqual(allparams, self.attgenparams)
def init(self): #MEMORY: encodes how entity is written + custom entity embeddings wencpg = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=self.glovepath) self.memenco = SeqEncoder( wencpg, GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)) entemb = VectorEmbed(indim=self.outdim, dim=self.entembdim) self.mempayload = ConcatBlock(entemb, self.memenco) self.memblock = MemoryBlock(self.mempayload, self.memdata, indim=self.outdim, outdim=self.encinnerdim + self.entembdim) #ENCODER: uses the same language encoder as memory #wencpg2 = WordEncoderPlusGlove(numchars=self.numchars, numwords=self.numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath) self.enc = RecStack( wencpg, GRU(dim=self.wordembdim + self.wordencdim, innerdim=self.encinnerdim)) #ATTENTION attgen = LinearGateAttentionGenerator(indim=self.encinnerdim + self.decinnerdim, innerdim=self.attdim) attcon = WeightedSumAttCon() #DECODER #entemb2 = VectorEmbed(indim=self.outdim, dim=self.entembdim) self.softmaxoutblock = stack( self.memaddr(self.memblock, indim=self.decinnerdim + self.encinnerdim, memdim=self.memblock.outdim, attdim=self.attdim), Softmax()) self.dec = SeqDecoder([ self.memblock, GRU(dim=self.entembdim + self.encinnerdim, innerdim=self.decinnerdim) ], outconcat=True, inconcat=False, attention=Attention(attgen, attcon), innerdim=self.decinnerdim + self.encinnerdim, softmaxoutblock=self.softmaxoutblock)
def setUp(self): criteriondim = 30 datadim = 20 innerdim = 25 batsize = 33 seqlen = 11 self.attgenshape = (batsize, seqlen) self.attconshape = (batsize, datadim) self.attgenc = self.getattgenc() self.attgen = self.attgenc(indim=criteriondim + datadim, innerdim=innerdim) self.attgenparams = self.getattgenparams() self.attcon = WeightedSumAttCon() self.att = Attention(self.attgen, self.attcon) self.criterion_val = np.random.random((batsize, criteriondim)).astype("float32") self.data_val = np.random.random((batsize, seqlen, datadim)).astype("float32")
def __init__(self, wordembdim=50, wordencdim=50, entembdim=200, innerdim=200, attdim=100, outdim=1e4, numwords=4e5, numchars=128, glovepath=None, **kw): super(FBSeqCompEncDecAtt, self).__init__(**kw) self.indim = wordembdim + wordencdim self.outdim = outdim self.wordembdim = wordembdim self.wordencdim = wordencdim self.encinnerdim = innerdim self.entembdim = entembdim self.decinnerdim = innerdim self.wordencoder = WordEncoderPlusGlove(numchars=numchars, numwords=numwords, encdim=self.wordencdim, embdim=self.wordembdim, embtrainfrac=0.0, glovepath=glovepath) self.rnn = RecStack( self.wordencoder, GRU(dim=wordembdim + wordencdim, innerdim=self.encinnerdim)) attgen = LinearGateAttentionGenerator(indim=self.encinnerdim + self.decinnerdim, innerdim=attdim) attcon = WeightedSumAttCon() self.dec = SeqDecoder([ VectorEmbed(indim=self.outdim, dim=self.entembdim), GRU(dim=self.entembdim, innerdim=self.decinnerdim) ], attention=Attention(attgen, attcon), outconcat=True, inconcat=False, innerdim=self.encinnerdim + self.decinnerdim)
def __init__(self, inpvocsize=None, inpembdim=None, inpemb=None, inpencinnerdim=None, bidir=False, maskid=None, dropout=False, rnu=GRU, inpencoder=None, memvocsize=None, memembdim=None, memembmat=None, memencinnerdim=None, memencoder=None, inp_att_dist=CosineDistance(), mem_att_dist=CosineDistance(), inp_attention=None, mem_attention=None, coredims=None, corernu=GRU, core=None, explicit_interface=False, scalaraggdim=None, write_value_dim=None, nsteps=100, posvecdim=None, mem_pos_repr=None, inp_pos_repr=None, inp_addr_extractor=None, mem_addr_extractor=None, write_addr_extractor=None, write_addr_generator=None, write_addr_dist=CosineDistance(), write_value_generator=None, write_value_extractor=None, mem_erase_generator=None, mem_change_generator=None, memsampler=None, memsamplemethod=None, memsampletemp=0.3, **kw): # INPUT ENCODING if inpencoder is None: inpencoder = SeqEncoder.RNN(indim=inpvocsize, inpembdim=inpembdim, inpemb=inpemb, innerdim=inpencinnerdim, bidir=bidir, maskid=maskid, dropout_in=dropout, dropout_h=dropout, rnu=rnu).all_outputs() lastinpdim = inpencinnerdim if not issequence( inpencinnerdim) else inpencinnerdim[-1] else: lastinpdim = inpencoder.block.layers[-1].innerdim # MEMORY ENCODING if memembmat is None: memembmat = param((memvocsize, memembdim), name="memembmat").glorotuniform() if memencoder is None: memencoder = SeqEncoder.RNN(inpemb=False, innerdim=memencinnerdim, bidir=bidir, dropout_in=dropout, dropout_h=dropout, rnu=rnu, inpembdim=memembdim).all_outputs() lastmemdim = memencinnerdim if not issequence( memencinnerdim) else memencinnerdim[-1] else: lastmemdim = memencoder.block.layers[-1].innerdim # POSITION VECTORS if posvecdim is not None and inp_pos_repr is None: inp_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout) if posvecdim is not None and mem_pos_repr is None: mem_pos_repr = RNNWithoutInput(posvecdim, dropout=dropout) xtra_dim = posvecdim if posvecdim is not None else 0 # CORE RNN - THE THINKER if core is None: corelayers, _ = MakeRNU.fromdims( [lastinpdim + lastmemdim + xtra_dim * 2] + coredims, rnu=corernu, dropout_in=dropout, dropout_h=dropout, param_init_states=True) core = RecStack(*corelayers) lastcoredim = core.get_statespec()[-1][0][1][0] # ATTENTIONS if mem_attention is None: mem_attention = Attention(mem_att_dist) if inp_attention is None: inp_attention = Attention(inp_att_dist) if write_addr_generator is None: write_addr_generator = AttGen(write_addr_dist) # WRITE VALUE if write_value_generator is None: write_value_generator = WriteValGenerator(write_value_dim, memvocsize, dropout=dropout) # MEMORY SAMPLER if memsampler is not None: assert (memsamplemethod is None) if memsamplemethod is not None: assert (memsampler is None) memsampler = GumbelSoftmax(temperature=memsampletemp) ################ STATE INTERFACES ################# if not explicit_interface: if inp_addr_extractor is None: inp_addr_extractor = Forward(lastcoredim, lastinpdim + xtra_dim, dropout=dropout) if mem_addr_extractor is None: inp_addr_extractor = Forward(lastcoredim, lastmemdim + xtra_dim, dropout=dropout) # WRITE INTERFACE if write_addr_extractor is None: write_addr_extractor = Forward(lastcoredim, lastmemdim + xtra_dim, dropout=dropout) if write_value_extractor is None: write_value_extractor = Forward(lastcoredim, write_value_dim, dropout=dropout) # MEM UPDATE INTERFACE if mem_erase_generator is None: mem_erase_generator = StateToScalar(lastcoredim, scalaraggdim) if mem_change_generator is None: mem_change_generator = StateToScalar(lastcoredim, scalaraggdim) else: inp_addr_extractor, mem_addr_extractor, write_addr_extractor, \ write_value_extractor, mem_erase_generator, mem_change_generator = \ make_vector_slicers(0, lastinpdim + xtra_dim, lastmemdim + xtra_dim, lastmemdim + xtra_dim, write_value_dim, 1, 1) super(SimpleBulkNN, self).__init__(inpencoder=inpencoder, memembmat=memembmat, memencoder=memencoder, inp_attention=inp_attention, mem_attention=mem_attention, core=core, memsampler=memsampler, nsteps=nsteps, inp_addr_extractor=inp_addr_extractor, mem_addr_extractor=mem_addr_extractor, write_addr_extractor=write_addr_extractor, write_addr_generator=write_addr_generator, mem_erase_generator=mem_erase_generator, mem_change_generator=mem_change_generator, write_value_generator=write_value_generator, write_value_extractor=write_value_extractor, inp_pos_repr=inp_pos_repr, mem_pos_repr=mem_pos_repr, **kw)
def _getattention(self, attdist, sepatt=False): attgen = AttGen(attdist) attcon = WeightedSumAttCon() attention = Attention(attgen, attcon, separate=sepatt) return attention